From 905def3419fc9aded4387e11a9e6c61caa5eb2c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 16:52:23 +0000 Subject: [PATCH 01/22] Postgres parser refuses deep nesting instead of exhausting the stack Every recursive path now passes through the parser's one depth counter: FROM items, function and JSON_OBJECT keys read outside the operator climb, statements inside BEGIN ATOMIC and EXPLAIN, and type names, whose [] bounds count against the same limit. Each level runs on a stack that grows on demand, so the 300-level limit, not the thread's stack size, decides. Once one reading passes the limit, every other reading fails at once and the statement is refused with 54001. Flat chains of operators other than AND and OR, casts and indirections are refused past 1,000 links. A parenthesized query and JSON_OBJECT's arguments are parsed once per start token and replayed, so nested IN subqueries and json_object calls no longer backtrack exponentially. --- Cargo.lock | 1 + crates/sqlquack-syntax-postgres/Cargo.toml | 6 + crates/sqlquack-syntax-postgres/README.md | 16 + .../src/parse/_helpers/commands/from.rs | 8 +- .../src/parse/_helpers/commands/queries.rs | 8 +- .../src/parse/_helpers/commands/routines.rs | 2 +- .../src/parse/_helpers/commands/utility.rs | 2 +- .../src/parse/_helpers/tree/cursor.rs | 118 ++++- .../src/parse/_helpers/tree/typenames.rs | 7 + .../src/parse/_helpers/values/expressions.rs | 8 +- .../src/parse/_helpers/values/primary.rs | 10 + .../src/parse/_helpers/values/sqljson.rs | 11 +- .../src/parse/main/parse.rs | 5 +- .../src/parse/main/parse_type.rs | 5 +- .../src/parse/tests.rs | 2 + .../src/parse/tests/test_depth.rs | 418 ++++++++++++++++++ .../src/parse/tests/test_types.rs | 14 + 17 files changed, 621 insertions(+), 20 deletions(-) create mode 100644 crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs diff --git a/Cargo.lock b/Cargo.lock index 31cd36964..33494142e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1521,6 +1521,7 @@ version = "0.2.0" dependencies = [ "serde_json", "sqlquack-syntax", + "stacker", ] [[package]] diff --git a/crates/sqlquack-syntax-postgres/Cargo.toml b/crates/sqlquack-syntax-postgres/Cargo.toml index 2ac6bc213..2ee32afb5 100644 --- a/crates/sqlquack-syntax-postgres/Cargo.toml +++ b/crates/sqlquack-syntax-postgres/Cargo.toml @@ -9,8 +9,14 @@ homepage.workspace = true publish = false rust-version.workspace = true +# Stack growth is the only feature; without it, deep nesting needs a large enough thread stack. +[features] +default = ["stacker"] +stacker = ["dep:stacker"] + [dependencies] sqlquack-syntax.workspace = true +stacker = { workspace = true, optional = true } [dev-dependencies] serde_json.workspace = true diff --git a/crates/sqlquack-syntax-postgres/README.md b/crates/sqlquack-syntax-postgres/README.md index 24541074c..86ddd5e23 100644 --- a/crates/sqlquack-syntax-postgres/README.md +++ b/crates/sqlquack-syntax-postgres/README.md @@ -9,6 +9,22 @@ carries the message, the byte position and whether the SQLSTATE is 42601. Other as an invalid Unicode escape (22025), the `UNIQUE` predicate (0A000) or the stack depth limit (54001), have `syntax = false`. +## Nesting + +Nesting is bounded so that no input exhausts the caller's stack or takes exponential time: + +- Each nested level (parentheses, subqueries, FROM items, calls, CASE, prefixes such as `NOT`, + statements inside `BEGIN ATOMIC` and `EXPLAIN`, and type names) runs on a stack that grows on + demand (the default `stacker` feature), and a statement nesting past 300 levels is refused with + 54001's "stack depth limit exceeded". A type's `[]` bounds count against the same limit. Once + one reading passes the limit, every other reading fails at once and the statement is refused. + PostgreSQL's own limit depends on its stack size, so it accepts some deeper texts. +- A flat chain of operators (other than `AND` and `OR`), casts or indirections is refused with + the same error past 1,000 links. `AND`, `OR` and set operations are not counted; their trees are + as deep as the chain, so a consumer that walks them recursively must bound the depth itself. +- A parenthesized query, and `JSON_OBJECT`'s arguments, are parsed once from each token and the + outcome replayed, so alternatives that each read the same nested text stay linear. + ## Gaps A statement holding a construct the shared tree cannot hold becomes a `Command` with the diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/from.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/from.rs index 7c1b572d2..776731e03 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/from.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/from.rs @@ -191,9 +191,11 @@ impl Parser<'_> { /// `table_ref` without its joins, with its source range. pub(crate) fn table_ref(&mut self) -> Parsing { - let first: usize = self.at; - let node: Expression = self.table_item()?; - Ok(self.spanned(first, node)) + self.deeper(|parser| { + let first: usize = parser.at; + let node: Expression = parser.table_item()?; + Ok(parser.spanned(first, node)) + }) } /// `table_ref` without its joins. diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/queries.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/queries.rs index 19a8942c0..a48ededd2 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/queries.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/queries.rs @@ -11,7 +11,7 @@ use sqlquack_syntax::expressions::statements::models::Lock; use crate::parse::_helpers::commands::tables::clause; use crate::parse::_helpers::tree::cursor::{ - Parser, Parsing, is_bare_label, is_col_id, is_col_label, + Parser, Parsing, Remembered, is_bare_label, is_col_id, is_col_label, }; use crate::parse::_helpers::tree::nodes::{ boolean, cte, function, keyword_call, literal, null, set_operation, star, subquery, table, @@ -67,6 +67,12 @@ fn selectable(body: Expression) -> Expression { impl Parser<'_> { /// `select_with_parens`: `( query )`. pub(crate) fn select_with_parens(&mut self) -> Parsing { + self.remember(Remembered::ParenthesizedQuery, |parser| { + parser.parenthesized_query() + }) + } + + fn parenthesized_query(&mut self) -> Parsing { self.expect_char('(')?; let query: Expression = self.query()?; self.expect_char(')')?; diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/routines.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/routines.rs index cf73e4839..524c6130e 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/routines.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/routines.rs @@ -766,7 +766,7 @@ impl Parser<'_> { } let statement: Expression = match self.eat_kw("return") { true => Expression::ReturnStmt(Box::new(self.a_expr()?)), - false => self.statement()?, + false => self.deeper(|parser| parser.statement())?, }; statements.push(statement); self.expect_char(';')?; diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/utility.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/utility.rs index 4600da969..02f7d736b 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/utility.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/commands/utility.rs @@ -408,7 +408,7 @@ impl Parser<'_> { let start: usize = self.at; let statement: Expression = match self.keyword_at(0) { Some(word) if EXPLAINED_STARTS.contains(&word) => { - let statement: Expression = self.statement()?; + let statement: Expression = self.deeper(|parser| parser.statement())?; let explainable: bool = match &statement { Expression::CreateTable(node) => node.as_select.is_some(), Expression::UtilityStatement(node) => { diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs index 8ef42bcbc..c91b43481 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs @@ -1,5 +1,8 @@ //! The parser's state: the tokens, where it is, and how tokens read as names. +use std::collections::HashMap; + +use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Identifier; use crate::keywords::models::KeywordCategory; @@ -13,11 +16,35 @@ pub(crate) type Parsing = Result; /// Recursion deeper than this fails as PostgreSQL's stack depth limit does (54001), not as syntax. const MAX_DEPTH: usize = 300; +/// The longest flat chain of operators other than AND and OR, or of indirections. +const MAX_CHAIN: usize = 1000; + +/// PostgreSQL's message for 54001, which recursion and chains past their limits both raise. +const STACK_DEPTH_EXCEEDED: &str = "stack depth limit exceeded"; + +/// A construct whose parse depends only on the token it starts at, so it is parsed once there. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) enum Remembered { + ParenthesizedQuery, + JsonObjectArguments, +} + +/// How a remembered construct's parse ended: its result, where it stopped, and the gaps it found. +struct Outcome { + result: Parsing, + reached: usize, + gaps: Vec, +} + pub(crate) struct Parser<'sql> { pub(crate) sql: &'sql str, pub(crate) tokens: Vec, pub(crate) at: usize, depth: usize, + /// The depth limit's error once any path, even one later backtracked, has gone past it. + too_deep: Option, + /// Constructs already parsed from a token, replayed when another alternative reaches them. + remembered: HashMap<(Remembered, usize), Outcome>, pub(crate) gaps: Vec, /// Whether the statement being parsed holds a gap, so it becomes a `Command`. pub(crate) opaque: bool, @@ -97,6 +124,8 @@ impl<'sql> Parser<'sql> { tokens, at: 0, depth: 0, + too_deep: None, + remembered: HashMap::new(), gaps: Vec::new(), opaque: false, } @@ -331,21 +360,96 @@ impl<'sql> Parser<'sql> { }); } - /// Runs `parse` one level deeper, failing past the depth limit as PostgreSQL does. + /// Runs `parse` one level deeper; past the limit it and every later call fail, as 54001. pub(crate) fn deeper(&mut self, parse: impl FnOnce(&mut Self) -> Parsing) -> Parsing { + if let Some(error) = &self.too_deep { + return Err(error.clone()); + } if self.depth >= MAX_DEPTH { - return Err(ParseError { - message: "stack depth limit exceeded".to_owned(), - position: self.offset(), - syntax: false, - }); + return Err(self.refuse_depth()); } self.depth += 1; - let result: Parsing = parse(self); + let result: Parsing = self.with_stack(parse); self.depth -= 1; result } + /// Refuses the statement when `levels` more levels here, such as array bounds, pass the limit. + pub(crate) fn within_depth_by(&mut self, levels: usize) -> Parsing<()> { + match self.depth + levels > MAX_DEPTH { + true => Err(self.refuse_depth()), + false => Ok(()), + } + } + + /// A flat chain's link count after one more link, refusing the statement past `MAX_CHAIN`. + pub(crate) fn link(&mut self, links: usize) -> Parsing { + match links < MAX_CHAIN { + true => Ok(links + 1), + false => Err(self.refuse_depth()), + } + } + + /// The depth limit's error, recorded so every later level and the statement fail with it. + fn refuse_depth(&mut self) -> ParseError { + let error: ParseError = ParseError { + message: STACK_DEPTH_EXCEEDED.to_owned(), + position: self.offset(), + syntax: false, + }; + self.too_deep.get_or_insert(error).clone() + } + + /// `parse` of `construct` from the cursor, or a replay of its earlier parse from there. + pub(crate) fn remember( + &mut self, + construct: Remembered, + parse: impl FnOnce(&mut Self) -> Parsing, + ) -> Parsing { + let key: (Remembered, usize) = (construct, self.at); + if let Some(outcome) = self.remembered.get(&key) { + self.at = outcome.reached; + self.opaque |= !outcome.gaps.is_empty(); + self.gaps.extend(outcome.gaps.iter().cloned()); + return outcome.result.clone(); + } + let gaps: usize = self.gaps.len(); + let result: Parsing = parse(self); + let outcome: Outcome = Outcome { + result: result.clone(), + reached: self.at, + gaps: self.gaps[gaps.min(self.gaps.len())..].to_vec(), + }; + self.remembered.insert(key, outcome); + result + } + + /// `result`, or the depth limit's error if any path went past it. + pub(crate) fn unless_too_deep(&self, result: Parsing) -> Parsing { + match &self.too_deep { + Some(error) => Err(error.clone()), + None => result, + } + } + + /// Runs `parse` on a grown stack when this one is nearly used up, whatever the thread's size. + #[inline(always)] + fn with_stack(&mut self, parse: impl FnOnce(&mut Self) -> Parsing) -> Parsing { + #[cfg(feature = "stacker")] + { + let red_zone: usize = match cfg!(debug_assertions) { + true => 4 * 1024 * 1024, + false => 1024 * 1024, + }; + stacker::maybe_grow(red_zone, 8 * 1024 * 1024, || parse(self)) + } + + #[cfg(not(feature = "stacker"))] + { + parse(self) + } + } + /// Tries `parse`; on failure rewinds and returns the error and how far it got. pub(crate) fn attempt( &mut self, diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/typenames.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/typenames.rs index a27777e0d..f40531757 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/typenames.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/typenames.rs @@ -125,6 +125,10 @@ fn integer(text: &str) -> Option { impl Parser<'_> { /// `Typename`: a simple type, then `[n]...` bounds or `ARRAY [ [n] ]`. pub(crate) fn typename(&mut self) -> Parsing { + self.deeper(|parser| parser.bounded_typename()) + } + + fn bounded_typename(&mut self) -> Parsing { if self.eat_kw("setof") { return Ok(setof(self.typename()?)); } @@ -147,7 +151,10 @@ impl Parser<'_> { dimension, }); } + let mut bounds: usize = 0; while self.is_char('[') { + bounds += 1; + self.within_depth_by(bounds)?; let dimension: Option = self.array_bound()?; data_type = DataType::Array { element_type: Box::new(data_type), diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/expressions.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/expressions.rs index 1ddceee78..2c5aa77b8 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/expressions.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/expressions.rs @@ -153,6 +153,7 @@ impl Parser<'_> { let operand: Expression = parser.prefix(flavor)?; let mut left: Expression = parser.spanned(first, operand); let mut pending: Option = None; + let mut links: usize = 0; while let Some(level) = parser.infix_level(flavor) { if level < min { break; @@ -161,6 +162,9 @@ impl Parser<'_> { return Err(parser.error()); } let (node, nonassociative) = parser.infix(left, level, flavor)?; + if !matches!(node, Expression::And(_) | Expression::Or(_)) { + links = parser.link(links)?; + } left = parser.spanned(first, node); pending = nonassociative.then_some(level); } @@ -200,9 +204,9 @@ impl Parser<'_> { }))) } "unique" if flavor == Flavor::A => self.unique_predicate(), - _ => self.c_expr(flavor), + _ => self.primary(flavor), }, - _ => self.c_expr(flavor), + _ => self.primary(flavor), } } diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/primary.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/primary.rs index 06f868b60..f9f048d3e 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/primary.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/primary.rs @@ -49,7 +49,13 @@ fn dot(this: Expression, field: Identifier) -> Expression { } impl Parser<'_> { + /// `c_expr` one level deeper, for callers outside the operator climb. pub(crate) fn c_expr(&mut self, flavor: Flavor) -> Parsing { + self.deeper(|parser| parser.primary(flavor)) + } + + /// `c_expr` at the current depth; only the operator climb, already one level deeper, calls it. + pub(crate) fn primary(&mut self, flavor: Flavor) -> Parsing { let Some(token) = self.peek().cloned() else { return Err(self.error()); }; @@ -318,7 +324,11 @@ impl Parser<'_> { /// `opt_indirection`: `.field`, `.*`, `[i]` and `[i:j]` after a primary. pub(crate) fn indirection(&mut self, mut this: Expression) -> Parsing { + let mut links: usize = 0; loop { + if self.is_char('.') || self.is_char('[') { + links = self.link(links)?; + } if self.is_char('.') { if self.char_at(1, '*') { self.at += 2; diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/sqljson.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/sqljson.rs index d533e61ae..dd342aa42 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/sqljson.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/values/sqljson.rs @@ -4,7 +4,9 @@ use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Identifier; use crate::parse::_helpers::commands::routines::var; -use crate::parse::_helpers::tree::cursor::{Parser, Parsing, identifier, is_col_id, is_col_label}; +use crate::parse::_helpers::tree::cursor::{ + Parser, Parsing, Remembered, identifier, is_col_id, is_col_label, +}; use crate::parse::_helpers::tree::nodes::{keyword_call, tuple}; use crate::parse::_helpers::values::expressions::Flavor; use crate::parse::constants::{JSON_ENCODINGS, JSON_QUERY_FUNCTION, QUERY_STARTS}; @@ -152,6 +154,13 @@ impl Parser<'_> { self.expect_char(')')?; return Ok(keyword_call("JSON_OBJECT", Vec::new(), options)); } + self.remember(Remembered::JsonObjectArguments, |parser| { + parser.json_object_forms() + }) + } + + /// `JSON_OBJECT`'s pairs, or the old argument list as `json_object`, after its `(`. + fn json_object_forms(&mut self) -> Parsing { self.alternatives(&[ |parser| { let pairs: Vec = diff --git a/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs b/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs index bd8a35401..82be80fb6 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs @@ -2,7 +2,7 @@ use sqlquack_syntax::Expression; -use crate::parse::_helpers::tree::cursor::Parser; +use crate::parse::_helpers::tree::cursor::{Parser, Parsing}; use crate::parse::_helpers::tree::nodes::command; use crate::parse::errors::ParseError; use crate::parse::models::Parsed; @@ -22,7 +22,8 @@ pub fn parse(sql: &str) -> Result { } let first: usize = parser.at; parser.opaque = false; - let statement: Expression = parser.statement()?; + let statement: Parsing = parser.statement(); + let statement: Expression = parser.unless_too_deep(statement)?; if !(parser.peek().is_none() || parser.is_char(';')) { return Err(parser.error()); } diff --git a/crates/sqlquack-syntax-postgres/src/parse/main/parse_type.rs b/crates/sqlquack-syntax-postgres/src/parse/main/parse_type.rs index 9815ebd0b..9f6be67c1 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/main/parse_type.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/main/parse_type.rs @@ -2,7 +2,7 @@ use sqlquack_syntax::DataType; -use crate::parse::_helpers::tree::cursor::Parser; +use crate::parse::_helpers::tree::cursor::{Parser, Parsing}; use crate::parse::errors::ParseError; use crate::scan::main::scan::scan; @@ -10,7 +10,8 @@ use crate::scan::main::scan::scan; pub fn parse_type(text: &str) -> Result { let mut parser: Parser<'_> = Parser::new(text, scan(text)); let setof: bool = parser.kw("setof"); - let data_type: DataType = parser.typename()?; + let data_type: Parsing = parser.typename(); + let data_type: DataType = parser.unless_too_deep(data_type)?; if parser.peek().is_some() { return Err(parser.error()); } diff --git a/crates/sqlquack-syntax-postgres/src/parse/tests.rs b/crates/sqlquack-syntax-postgres/src/parse/tests.rs index 0e6091417..6db5f2efe 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/tests.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_depth.rs"] +mod test_depth; #[path = "tests/test_parse.rs"] mod test_parse; #[path = "tests/test_types.rs"] diff --git a/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs new file mode 100644 index 000000000..3907e7780 --- /dev/null +++ b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs @@ -0,0 +1,418 @@ +//! Deep nesting in the PostgreSQL parser: refused at the depth limit, never by the thread's stack. + +use crate::parse::main::parse::parse; +use crate::parse::main::parse_type::parse_type; +use crate::parse::tests::test_types::NestingTestCase; + +/// A thread stack far smaller than the depth limit needs, so passing shows the stack grows on demand. +const SMALL_STACK_BYTES: usize = 256 * 1024; + +#[test] +fn given_deep_nesting_when_parsing_on_a_small_stack_then_the_depth_limit_refuses_it() { + let test_cases = [ + NestingTestCase { + description: "parenthesized joins in FROM", + prefix: "SELECT * FROM ", + open: "(", + innermost: "a JOIN b ON true", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "derived tables in FROM", + prefix: "SELECT * FROM ", + open: "(SELECT * FROM ", + innermost: "t", + close: ") s", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "LATERAL subqueries in FROM", + prefix: "SELECT * FROM t, ", + open: "LATERAL (SELECT * FROM t, ", + innermost: "u", + close: ") s", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "scalar subqueries", + prefix: "SELECT ", + open: "(SELECT ", + innermost: "1", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "EXISTS subqueries", + prefix: "SELECT ", + open: "EXISTS (SELECT ", + innermost: "1", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "IN subqueries", + prefix: "SELECT 1 IN ", + open: "(SELECT 1 IN (", + innermost: "SELECT 1", + close: "))", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "parenthesized queries", + prefix: "", + open: "(", + innermost: "SELECT 1", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "WITH queries", + prefix: "", + open: "WITH a AS (", + innermost: "SELECT 1", + close: ") SELECT 1", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "parenthesized expressions", + prefix: "SELECT ", + open: "(", + innermost: "1", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "function calls", + prefix: "SELECT ", + open: "abs(", + innermost: "1", + close: ")", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "CASE expressions", + prefix: "SELECT ", + open: "CASE WHEN true THEN ", + innermost: "1", + close: " END", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "array constructors", + prefix: "SELECT ", + open: "ARRAY[", + innermost: "1", + close: "]", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "unary minus", + prefix: "SELECT ", + open: "- ", + innermost: "1", + close: "", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "JSON_OBJECT keys, read as c_expr outside the operator climb", + prefix: "SELECT ", + open: "json_object(", + innermost: "1", + close: " VALUE 1)", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "parenthesized type modifiers in a cast", + prefix: "SELECT CAST(1 AS ", + open: "t(", + innermost: "int", + close: "))", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "BEGIN ATOMIC function bodies", + prefix: "", + open: "CREATE FUNCTION f() RETURNS int BEGIN ATOMIC ", + innermost: " SELECT 1;", + close: " END;", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "parenthesized expressions well inside the limit", + prefix: "SELECT ", + open: "(", + innermost: "1", + close: ")", + read: |text| parse(text).err(), + levels: 100, + expected_error: None, + }, + NestingTestCase { + description: "derived tables well inside the limit", + prefix: "SELECT * FROM ", + open: "(SELECT * FROM ", + innermost: "t", + close: ") s", + read: |text| parse(text).err(), + levels: 40, + expected_error: None, + }, + NestingTestCase { + description: "old-style json_object lists, whose three readings each hold the next", + prefix: "SELECT ", + open: "json_object(", + innermost: "1", + close: ", 1)", + read: |text| parse(text).err(), + levels: 60, + expected_error: None, + }, + NestingTestCase { + description: "unclosed IN ((SELECT, each also read as a list holding the next", + prefix: "SELECT 1 IN (", + open: "(SELECT 1 IN (", + innermost: "", + close: "", + read: |text| parse(text).err(), + levels: 60, + expected_error: Some(("syntax error at end of input", true)), + }, + NestingTestCase { + description: "unclosed json_object calls, each read three ways", + prefix: "SELECT ", + open: "json_object(", + innermost: "", + close: "", + read: |text| parse(text).err(), + levels: 60, + expected_error: Some(("syntax error at end of input", true)), + }, + NestingTestCase { + description: "100 nested FROM subqueries, which a 2 MiB thread could not hold before", + prefix: "SELECT * FROM ", + open: "(SELECT * FROM ", + innermost: "t", + close: ") s", + read: |text| parse(text).err(), + levels: 100, + expected_error: None, + }, + NestingTestCase { + description: "parenthesized tables in FROM", + prefix: "SELECT * FROM ", + open: "(", + innermost: "t", + close: ")", + read: |text| parse(text).err(), + levels: 3000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "NOT prefixes", + prefix: "SELECT ", + open: "NOT ", + innermost: "true", + close: "", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "nested subscripts", + prefix: "SELECT ", + open: "a[", + innermost: "1", + close: "]", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "[] suffixes on a cast type", + prefix: "SELECT 1::int", + open: "[]", + innermost: "", + close: "", + read: |text| parse(text).err(), + levels: 100000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "[3] suffixes on a CAST type", + prefix: "SELECT CAST(1 AS int", + open: "[3]", + innermost: "", + close: ")", + read: |text| parse(text).err(), + levels: 100000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "[] suffixes on a column type", + prefix: "CREATE TABLE t (a int", + open: "[]", + innermost: "", + close: ")", + read: |text| parse(text).err(), + levels: 100000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "casts nesting array types, each type 60 bounds deep, counted together", + prefix: "SELECT ", + open: "CAST(", + innermost: "1", + close: " AS int[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][])", + read: |text| parse(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "casts nesting array types, few enough levels in all", + prefix: "SELECT ", + open: "CAST(", + innermost: "1", + close: " AS int[][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][][])", + read: |text| parse(text).err(), + levels: 20, + expected_error: None, + }, + NestingTestCase { + description: "1,000 additions, the longest chain", + prefix: "SELECT 1", + open: "", + innermost: "", + close: " + 1", + read: |text| parse(text).err(), + levels: 1000, + expected_error: None, + }, + NestingTestCase { + description: "1,001 additions", + prefix: "SELECT 1", + open: "", + innermost: "", + close: " + 1", + read: |text| parse(text).err(), + levels: 1001, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "100,000 casts in a chain", + prefix: "SELECT 1", + open: "", + innermost: "", + close: "::int", + read: |text| parse(text).err(), + levels: 100000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "100,000 field selections", + prefix: "SELECT (a)", + open: "", + innermost: "", + close: ".b", + read: |text| parse(text).err(), + levels: 100000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "AND chains are not counted", + prefix: "SELECT true", + open: "", + innermost: "", + close: " AND true", + read: |text| parse(text).err(), + levels: 5000, + expected_error: None, + }, + NestingTestCase { + description: "set operations are not counted", + prefix: "SELECT 1", + open: "", + innermost: "", + close: " UNION ALL SELECT 1", + read: |text| parse(text).err(), + levels: 5000, + expected_error: None, + }, + NestingTestCase { + description: "parenthesized type modifiers in a type string", + prefix: "", + open: "t(", + innermost: "int", + close: ")", + read: |text| parse_type(text).err(), + levels: 2000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "parenthesized type modifiers in a type string well inside the limit", + prefix: "", + open: "t(", + innermost: "int", + close: ")", + read: |text| parse_type(text).err(), + levels: 40, + expected_error: None, + }, + ]; + for test_case in test_cases { + let text: String = format!( + "{}{}{}{}", + test_case.prefix, + test_case.open.repeat(test_case.levels), + test_case.innermost, + test_case.close.repeat(test_case.levels), + ); + let read = test_case.read; + let error: Option<(String, bool)> = std::thread::Builder::new() + .stack_size(SMALL_STACK_BYTES) + .spawn(move || read(&text).map(|error| (error.message, error.syntax))) + .expect("the parsing thread starts") + .join() + .expect("parsing returns rather than panicking"); + assert_eq!( + error, + test_case + .expected_error + .map(|(message, syntax)| (message.to_owned(), syntax)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-syntax-postgres/src/parse/tests/test_types.rs b/crates/sqlquack-syntax-postgres/src/parse/tests/test_types.rs index a481c806c..3236b3f02 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/tests/test_types.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/tests/test_types.rs @@ -1,5 +1,7 @@ //! Test case types for the PostgreSQL parser. +use crate::parse::errors::ParseError; + /// What PostgreSQL 18.6's raw parser does with an input. #[derive(Debug, PartialEq, Eq)] pub(crate) enum ExpectedVerdict { @@ -62,3 +64,15 @@ pub(crate) struct GapTestCase { pub(crate) faulted: bool, pub(crate) expected_gap: Option, } + +/// A nesting built from its parts, the entry point that reads it, and its `(message, syntax)`. +pub(crate) struct NestingTestCase { + pub(crate) description: &'static str, + pub(crate) prefix: &'static str, + pub(crate) open: &'static str, + pub(crate) innermost: &'static str, + pub(crate) close: &'static str, + pub(crate) read: fn(&str) -> Option, + pub(crate) levels: usize, + pub(crate) expected_error: Option<(&'static str, bool)>, +} From 46b19ba93a54717ba3cc7bafbf280013b1f2319e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:06:01 +0000 Subject: [PATCH 02/22] fix: bound type nesting and serialize long operator chains without stack overflow A data type nested past 512 levels is refused with the bracket-nesting guard (SQ0103). Every level counts toward one depth: [] and [n] suffixes, ARRAY<...>, and ARRAY(, OBJECT(, MAP( and STRUCT( levels, so alternating short runs of each can no longer nest without bound. Suffix runs are checked as they are built, so 100,000 [] suffixes stop at level 513. Serializing a binary operator's operands and a data type's nested types moves to a fresh stack segment when the stack runs low, so a flat chain of 100,000 + terms and a 512-level type serialize on a 2 MiB thread. The JSON is unchanged. Dropping such chains was already iterative. The guard tests now serialize every tree they parse, and cover these spellings at 512, 513, 20,000 and 100,000 levels and a 100,000-term + chain on a 2 MiB thread. --- .../src/expressions/operators/models.rs | 6 + .../tree/main/_serialize_on_grown_stack.rs | 24 ++++ .../src/expressions/tree/main/mod.rs | 1 + .../src/expressions/types/models.rs | 82 ++++++++++++ crates/sqlquack-syntax/src/guard/constants.rs | 3 + .../src/guard/tests/helpers.rs | 28 ++++- .../src/guard/tests/test_ast_guard.rs | 117 +++++++++++++++++- .../src/guard/tests/test_types.rs | 12 ++ .../parser/_helpers/types/cast_base_types.rs | 10 +- .../src/parser/_helpers/types/casts.rs | 7 +- .../src/parser/_helpers/types/data_types.rs | 46 +++++-- 11 files changed, 306 insertions(+), 30 deletions(-) create mode 100644 crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs diff --git a/crates/sqlquack-syntax/src/expressions/operators/models.rs b/crates/sqlquack-syntax/src/expressions/operators/models.rs index b5fdcf95a..2f7e95cc4 100644 --- a/crates/sqlquack-syntax/src/expressions/operators/models.rs +++ b/crates/sqlquack-syntax/src/expressions/operators/models.rs @@ -140,7 +140,13 @@ pub struct Case { /// The payload of every binary operator variant: arithmetic, comparison, logical, bitwise. #[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BinaryOp { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] pub left: Expression, + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] pub right: Expression, /// Comments after the left operand (before the operator) #[serde(default)] diff --git a/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs b/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs new file mode 100644 index 000000000..75de2777f --- /dev/null +++ b/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs @@ -0,0 +1,24 @@ +//! `serialize_on_grown_stack`: serializes a recursive field on a fresh stack segment when the stack runs low. + +use serde::Serialize; +use serde::Serializer; + +/// Serializes `value`, moving to a fresh stack segment first when the current stack is nearly used. +pub(in crate::expressions) fn serialize_on_grown_stack( + value: &T, + serializer: S, +) -> Result +where + T: Serialize + ?Sized, + S: Serializer, +{ + #[cfg(feature = "stacker")] + { + stacker::maybe_grow(256 * 1024, 4 * 1024 * 1024, || value.serialize(serializer)) + } + + #[cfg(not(feature = "stacker"))] + { + value.serialize(serializer) + } +} diff --git a/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs b/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs index 9775f4978..7c56a7206 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs @@ -1,3 +1,4 @@ //! Main. +pub(in crate::expressions) mod _serialize_on_grown_stack; pub(in crate::expressions) mod _type_assertion_result; diff --git a/crates/sqlquack-syntax/src/expressions/types/models.rs b/crates/sqlquack-syntax/src/expressions/types/models.rs index 7aa4bfa4a..f9e4639d7 100644 --- a/crates/sqlquack-syntax/src/expressions/types/models.rs +++ b/crates/sqlquack-syntax/src/expressions/types/models.rs @@ -133,6 +133,9 @@ pub enum OracleTimestampTimeZone { #[serde(tag = "data_type", rename_all = "snake_case")] pub enum DataType { Vertica { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] vertica_type: Box, }, Boolean, @@ -250,6 +253,9 @@ pub enum DataType { Uuid, Array { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] element_type: Box, /// Optional dimension size for PostgreSQL (e.g., [3] in INT[3]) #[serde(default, skip_serializing_if = "Option::is_none")] @@ -258,15 +264,27 @@ pub enum DataType { /// Materialize postfix LIST type, such as `INT LIST`. List { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] element_type: Box, }, Struct { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] fields: Vec, nested: bool, }, Map { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] key_type: Box, + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] value_type: Box, }, @@ -281,26 +299,41 @@ pub enum DataType { }, Union { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] fields: Vec<(String, DataType)>, }, Vector { #[serde(default)] + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] element_type: Option>, dimension: Option, }, Object { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] fields: Vec<(String, DataType, bool)>, modifier: Option, }, Nullable { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] inner: Box, }, /// An element or value type declared NOT NULL, as in Snowflake's `ARRAY ( INT NOT NULL )`. NotNull { + #[serde( + serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" + )] inner: Box, }, @@ -332,6 +365,55 @@ pub struct HanaDataType { } impl DataType { + /// The longest chain of types nested inside this one: 0 for `INT`, 2 for `INT[][]`. + pub(crate) fn nesting_depth(&self) -> usize { + let mut deepest = 0; + let mut pending: Vec<(&DataType, usize)> = vec![(self, 0)]; + while let Some((data_type, depth)) = pending.pop() { + deepest = deepest.max(depth); + let next = depth + 1; + match data_type { + DataType::Array { element_type, .. } + | DataType::List { element_type } + | DataType::Vector { + element_type: Some(element_type), + .. + } + | DataType::Nullable { + inner: element_type, + } + | DataType::NotNull { + inner: element_type, + } => pending.push((element_type, next)), + DataType::Map { + key_type, + value_type, + } => pending.extend([(key_type.as_ref(), next), (value_type.as_ref(), next)]), + DataType::Struct { fields, .. } => { + pending.extend(fields.iter().map(|field| (&field.data_type, next))); + } + DataType::Object { fields, .. } => { + pending.extend(fields.iter().map(|(_, field, _)| (field, next))); + } + DataType::Union { fields } => { + pending.extend(fields.iter().map(|(_, field)| (field, next))); + } + DataType::Vertica { vertica_type } => match vertica_type.as_ref() { + VerticaDataType::Array { element_type, .. } + | VerticaDataType::Set { element_type, .. } => { + pending.push((element_type, next)); + } + VerticaDataType::Row { fields } => { + pending.extend(fields.iter().map(|field| (&field.data_type, next))); + } + _ => {} + }, + _ => {} + } + } + deepest + } + /// Whether this type is [`DataType::Unknown`] or has an unknown part, as `ARRAY`. pub fn contains_unknown(&self) -> bool { match self { diff --git a/crates/sqlquack-syntax/src/guard/constants.rs b/crates/sqlquack-syntax/src/guard/constants.rs index 343d44fe9..14d13c576 100644 --- a/crates/sqlquack-syntax/src/guard/constants.rs +++ b/crates/sqlquack-syntax/src/guard/constants.rs @@ -12,6 +12,9 @@ pub(crate) const DEFAULT_MAX_PARENTHESES_DEPTH: usize = 512; pub(crate) const DEFAULT_MAX_FUNCTION_CALL_DEPTH: usize = 64; +/// Deepest type nesting, every `[]`, `ARRAY<`, `ARRAY(`, `OBJECT(` or `MAP(` level counted as one. +pub(crate) const MAX_DATA_TYPE_DEPTH: usize = 512; + #[cfg(target_arch = "wasm32")] pub(crate) const DEFAULT_MAX_PARSER_DEPTH: usize = 32; diff --git a/crates/sqlquack-syntax/src/guard/tests/helpers.rs b/crates/sqlquack-syntax/src/guard/tests/helpers.rs index 6373cfe21..325eb1ba3 100644 --- a/crates/sqlquack-syntax/src/guard/tests/helpers.rs +++ b/crates/sqlquack-syntax/src/guard/tests/helpers.rs @@ -62,14 +62,32 @@ pub(crate) fn guard_json_accepted(json: &str) -> bool { serde_json::from_str::(json).is_ok() } -/// The default-guarded Snowflake parse of `sql`, or its error message. +/// The default-guarded Snowflake parse of `sql`, serialized as JSON, or its error message. pub(crate) fn guarded_parse(sql: &str) -> Result<(), String> { - crate::parse_statements_with( + guarded_parse_in(sql, crate::Dialect::Snowflake) +} + +/// The default-guarded parse of `sql` in `dialect`, serialized as JSON, or its error message. +pub(crate) fn guarded_parse_in(sql: &str, dialect: crate::Dialect) -> Result<(), String> { + let statements = crate::parse_statements_with( sql, - crate::Dialect::Snowflake, + dialect, &crate::ParseOptions::default(), crate::CallMode::Faithful, ) - .map(|_| ()) - .map_err(|error| error.to_string()) + .map_err(|error| error.to_string())?; + serde_json::to_string(&statements) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +/// `n` type levels: constructor levels that each wrap 60 `[]` suffixes, then the remaining suffixes. +pub(crate) fn constructor_levels(n: usize, open: &str) -> String { + let levels = n / 61; + format!( + "SELECT CAST(NULL AS {}INT{}{})", + open.repeat(levels), + format!("{})", "[]".repeat(60)).repeat(levels), + "[]".repeat(n - 61 * levels) + ) } diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index 2b3ed2824..fd204bb0d 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -1,5 +1,9 @@ +use crate::guard::tests::helpers::constructor_levels; use crate::guard::tests::helpers::guarded_parse; +use crate::guard::tests::helpers::guarded_parse_in; use crate::guard::tests::test_types::AstDepthTestCase; +use crate::guard::tests::test_types::TypeDepthTestCase; +use crate::guard::tests::test_types::TYPE_DEPTHS; #[test] fn given_a_long_flat_operator_chain_when_parsing_then_only_real_nesting_counts_as_depth() { @@ -39,7 +43,7 @@ fn given_a_long_flat_operator_chain_when_parsing_then_only_real_nesting_counts_a } #[test] -fn given_a_70k_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_refused_without_overflow( +fn given_a_long_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_refused_without_overflow( ) { let test_cases = [ AstDepthTestCase { @@ -47,6 +51,11 @@ fn given_a_70k_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_ref sql: || format!("SELECT 1{}", "+1".repeat(70_000)), expected_error: None, }, + AstDepthTestCase { + description: "a 100,000-term + chain is flat, parses and serializes", + sql: || format!("SELECT 1{}", "+1".repeat(100_000)), + expected_error: None, + }, AstDepthTestCase { description: "a || chain is flat and parses", sql: || format!("SELECT 'a'{}", "||'a'".repeat(70_000)), @@ -122,3 +131,109 @@ fn given_a_70k_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_ref ); } } + +#[test] +fn given_a_deeply_nested_type_when_parsing_on_a_2_mb_thread_then_512_levels_parse_and_more_are_refused( +) { + const NESTING: Option<&str> = Some("E_GUARD_NESTING_DEPTH_EXCEEDED"); + const PARSER_DEPTH: Option<&str> = Some("E_GUARD_PARSER_DEPTH_EXCEEDED"); + const FUNCTION_NESTING: Option<&str> = Some("E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED"); + let test_cases = [ + TypeDepthTestCase { + description: "[] suffixes in a Snowflake CAST", + dialect: crate::Dialect::Snowflake, + sql: |n| format!("SELECT CAST(NULL AS INT{})", "[]".repeat(n)), + expected_errors: [None, NESTING, NESTING, NESTING], + }, + TypeDepthTestCase { + description: "[3] suffixes in a DuckDB CAST", + dialect: crate::Dialect::DuckDB, + sql: |n| format!("SELECT CAST(NULL AS INT{})", "[3]".repeat(n)), + expected_errors: [None, NESTING, NESTING, NESTING], + }, + TypeDepthTestCase { + description: "[] suffixes in a DuckDB :: cast", + dialect: crate::Dialect::DuckDB, + sql: |n| format!("SELECT x::INT{} FROM t", "[]".repeat(n)), + expected_errors: [None, NESTING, NESTING, NESTING], + }, + TypeDepthTestCase { + description: "[] suffixes on a Snowflake column type", + dialect: crate::Dialect::Snowflake, + sql: |n| format!("CREATE TABLE t (c INT{})", "[]".repeat(n)), + expected_errors: [None, NESTING, NESTING, NESTING], + }, + TypeDepthTestCase { + description: "nested ARRAY<> in a Snowflake CAST", + dialect: crate::Dialect::Snowflake, + sql: |n| { + format!( + "SELECT CAST(NULL AS {}INT{})", + "ARRAY<".repeat(n), + ">".repeat(n) + ) + }, + expected_errors: [None, NESTING, PARSER_DEPTH, PARSER_DEPTH], + }, + TypeDepthTestCase { + description: "ARRAY<> levels alternating with short [] runs", + dialect: crate::Dialect::Snowflake, + sql: |n| { + let levels = n / 4; + let inner = "[][][]>".repeat(levels); + let outer = "[]".repeat(n - 4 * levels); + format!( + "SELECT CAST(NULL AS {}INT{inner}{outer})", + "ARRAY<".repeat(levels) + ) + }, + expected_errors: [None, NESTING, PARSER_DEPTH, PARSER_DEPTH], + }, + TypeDepthTestCase { + description: "ARRAY( levels each wrapping 60 [] suffixes", + dialect: crate::Dialect::Snowflake, + sql: |n| constructor_levels(n, "ARRAY("), + expected_errors: [None, NESTING, FUNCTION_NESTING, FUNCTION_NESTING], + }, + TypeDepthTestCase { + description: "OBJECT( levels each wrapping 60 [] suffixes", + dialect: crate::Dialect::Snowflake, + sql: |n| constructor_levels(n, "OBJECT(a "), + expected_errors: [None, NESTING, FUNCTION_NESTING, FUNCTION_NESTING], + }, + TypeDepthTestCase { + description: "MAP( values each wrapping 60 [] suffixes", + dialect: crate::Dialect::Snowflake, + sql: |n| constructor_levels(n, "MAP(VARCHAR, "), + expected_errors: [None, NESTING, FUNCTION_NESTING, FUNCTION_NESTING], + }, + TypeDepthTestCase { + description: "STRUCT( levels each wrapping 60 [] suffixes", + dialect: crate::Dialect::DuckDB, + sql: |n| constructor_levels(n, "STRUCT(a "), + expected_errors: [None, NESTING, FUNCTION_NESTING, FUNCTION_NESTING], + }, + ]; + for test_case in test_cases { + for (index, levels) in TYPE_DEPTHS.into_iter().enumerate() { + let sql = (test_case.sql)(levels); + let dialect = test_case.dialect; + let outcome = std::thread::Builder::new() + .stack_size(2 << 20) + .spawn(move || guarded_parse_in(&sql, dialect)) + .expect("the parse thread starts") + .join() + .expect("the parse thread finishes"); + let code = outcome.as_ref().err().and_then(|error| { + error + .split(|c: char| !(c.is_ascii_uppercase() || c == '_')) + .find(|word| word.starts_with("E_")) + }); + assert_eq!( + code, test_case.expected_errors[index], + "{} at {levels} levels: {outcome:?}", + test_case.description + ); + } + } +} diff --git a/crates/sqlquack-syntax/src/guard/tests/test_types.rs b/crates/sqlquack-syntax/src/guard/tests/test_types.rs index 0f10bc7dc..b49c13886 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_types.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_types.rs @@ -38,3 +38,15 @@ pub(crate) struct AstDepthTestCase { /// None when the default guard accepts the query; otherwise a fragment of the error. pub(crate) expected_error: Option<&'static str>, } + +pub(crate) struct TypeDepthTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: crate::Dialect, + /// The query with its type nested `n` levels deep. + pub(crate) sql: fn(n: usize) -> String, + /// The guard code at each of [`TYPE_DEPTHS`] levels, or None where the parse succeeds. + pub(crate) expected_errors: [Option<&'static str>; 4], +} + +/// The nesting depths each [`TypeDepthTestCase`] is parsed at. +pub(crate) const TYPE_DEPTHS: [usize; 4] = [512, 513, 20_000, 100_000]; diff --git a/crates/sqlquack-syntax/src/parser/_helpers/types/cast_base_types.rs b/crates/sqlquack-syntax/src/parser/_helpers/types/cast_base_types.rs index f10493f83..e5ed3da3a 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/types/cast_base_types.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/types/cast_base_types.rs @@ -141,17 +141,11 @@ impl Parser { if self.match_token(TokenType::Lt) { let element_type = self.parse_data_type()?; self.expect_gt()?; - DataType::Array { - element_type: Box::new(element_type), - dimension: None, - } + self.array_type(element_type, None, None)? } else if self.match_token(TokenType::LParen) { let element_type = self.parse_data_type_for_cast()?; self.expect_token(TokenType::RParen)?; - DataType::Array { - element_type: Box::new(element_type), - dimension: None, - } + self.array_type(element_type, None, None)? } else { DataType::Custom { name } } diff --git a/crates/sqlquack-syntax/src/parser/_helpers/types/casts.rs b/crates/sqlquack-syntax/src/parser/_helpers/types/casts.rs index a1a4703d2..8a114eb46 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/types/casts.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/types/casts.rs @@ -364,6 +364,7 @@ impl Parser { /// Parse a data type. pub(crate) fn parse_data_type(&mut self) -> Result { let data_type = self.with_parser_depth(|parser| parser.parse_data_type_inner())?; + self.within_type_depth(data_type.nesting_depth())?; Ok(self.normalize_dialect_data_type(data_type)) } @@ -1325,10 +1326,7 @@ impl Parser { } else { self.expect_token(TokenType::RParen)?; } - element = DataType::Array { - element_type: Box::new(element), - dimension: None, - }; + element = self.array_type(element, None, None)?; if !closes.is_empty() { element = self.finish_data_type(element)?; scopes.pop(); @@ -1485,6 +1483,7 @@ impl Parser { /// Parses a `::TYPE` cast type; DuckDB reads `INT[3]` as a fixed-size array. pub(crate) fn parse_data_type_for_cast(&mut self) -> Result { let data_type = self.with_parser_depth(|parser| parser.parse_data_type_for_cast_inner())?; + self.within_type_depth(data_type.nesting_depth())?; Ok(self.normalize_dialect_data_type(data_type)) } } diff --git a/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs b/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs index 55bfc3698..5227bc166 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs @@ -12,6 +12,7 @@ use crate::expressions::types::models::OracleCharacterLengthSemantics; use crate::expressions::types::models::OracleDataType; use crate::expressions::types::models::OracleTimestampTimeZone; use crate::expressions::types::models::StructField; +use crate::guard::constants::MAX_DATA_TYPE_DEPTH; use crate::parser::constants::ARRAY; use crate::parser::constants::EXCLUDE; use crate::parser::constants::INCLUDE; @@ -541,9 +542,37 @@ impl Parser { } } + /// Refuses a type nested `depth` levels deep, counting every array, row and map level as one. + pub(crate) fn within_type_depth(&self, depth: usize) -> Result<()> { + if depth <= MAX_DATA_TYPE_DEPTH { + return Ok(()); + } + Err(Error::parse( + format!( + "E_GUARD_NESTING_DEPTH_EXCEEDED: value {depth} exceeds configured limit {MAX_DATA_TYPE_DEPTH}" + ), + self.previous().span, + )) + } + + /// An array of `element`, whose nesting depth is `element_depth` when known, up to the limit. + pub(crate) fn array_type( + &self, + element: DataType, + dimension: Option, + element_depth: Option, + ) -> Result { + self.within_type_depth(element_depth.unwrap_or_else(|| element.nesting_depth()) + 1)?; + Ok(DataType::Array { + element_type: Box::new(element), + dimension, + }) + } + /// Parse optional array dimensions after a type: [], [N], [N][M], ARRAY, ARRAY[N], etc. pub(crate) fn maybe_parse_array_dimensions(&mut self, base_type: DataType) -> Result { let mut current_type = base_type; + let mut depth: Option = None; if self.check_identifier("ARRAY") { self.skip(); @@ -559,10 +588,8 @@ impl Parser { } else { None }; - current_type = DataType::Array { - element_type: Box::new(current_type), - dimension, - }; + current_type = self.array_type(current_type, dimension, depth)?; + depth = Some(depth.map_or_else(|| current_type.nesting_depth(), |known| known + 1)); } while self.match_token(TokenType::LBracket) { @@ -574,10 +601,8 @@ impl Parser { }; self.expect_token(TokenType::RBracket)?; - current_type = DataType::Array { - element_type: Box::new(current_type), - dimension, - }; + current_type = self.array_type(current_type, dimension, depth)?; + depth = Some(depth.map_or_else(|| current_type.nesting_depth(), |known| known + 1)); } Ok(current_type) @@ -704,10 +729,7 @@ impl Parser { if self.match_token(TokenType::Lt) { let element_type = self.parse_data_type()?; self.expect_gt()?; - Ok(DataType::Array { - element_type: Box::new(element_type), - dimension: None, - }) + self.array_type(element_type, None, None) } else { Ok(DataType::Custom { name: "ARRAY".to_string(), From 2fc25a8f85d11faae82c1610920e1348dec20dc0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:00:40 +0000 Subject: [PATCH 03/22] fix: clone long operator chains without stack overflow The faithful-call cache clones each parsed call, and the derived Clone of a binary operator recursed once per level, so a few thousand + terms inside a function call overflowed a 2 MiB stack. Cloning an operator now moves to a fresh stack segment when the stack runs low, the same way its serialization does. The 2 MiB chain test gains 50,000-term + chains as the argument of f, SUM, COALESCE and ARRAY_CONSTRUCT. --- .../src/expressions/operators/models.rs | 27 ++++++++++++++++++- .../expressions/tree/main/_on_grown_stack.rs | 14 ++++++++++ .../tree/main/_serialize_on_grown_stack.rs | 12 +++------ .../src/expressions/tree/main/mod.rs | 1 + .../src/guard/tests/test_ast_guard.rs | 20 ++++++++++++++ 5 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 crates/sqlquack-syntax/src/expressions/tree/main/_on_grown_stack.rs diff --git a/crates/sqlquack-syntax/src/expressions/operators/models.rs b/crates/sqlquack-syntax/src/expressions/operators/models.rs index 2f7e95cc4..62fab949e 100644 --- a/crates/sqlquack-syntax/src/expressions/operators/models.rs +++ b/crates/sqlquack-syntax/src/expressions/operators/models.rs @@ -7,6 +7,7 @@ use serde::Serialize; use crate::expressions::names::models::Identifier; use crate::expressions::names::models::TableRef; use crate::expressions::queries::models::Ordered; +use crate::expressions::tree::main::_on_grown_stack::on_grown_stack; use crate::expressions::tree::models::Expression; use crate::expressions::types::models::DataType; @@ -138,7 +139,7 @@ pub struct Case { } /// The payload of every binary operator variant: arithmetic, comparison, logical, bitwise. -#[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(sqlquack_syntax_derive::AstNode, Debug, PartialEq, Serialize, Deserialize)] pub struct BinaryOp { #[serde( serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" @@ -167,6 +168,30 @@ pub struct BinaryOp { pub span: SourceSpan, } +/// Clones each operand on a grown stack when needed, so cloning a long chain cannot overflow. +impl Clone for BinaryOp { + fn clone(&self) -> Self { + let Self { + left, + right, + left_comments, + operator_comments, + trailing_comments, + inferred_type, + span, + } = self; + on_grown_stack(|| Self { + left: left.clone(), + right: right.clone(), + left_comments: left_comments.clone(), + operator_comments: operator_comments.clone(), + trailing_comments: trailing_comments.clone(), + inferred_type: inferred_type.clone(), + span: *span, + }) + } +} + impl BinaryOp { pub fn new(left: Expression, right: Expression) -> Self { Self { diff --git a/crates/sqlquack-syntax/src/expressions/tree/main/_on_grown_stack.rs b/crates/sqlquack-syntax/src/expressions/tree/main/_on_grown_stack.rs new file mode 100644 index 000000000..d1f7efd24 --- /dev/null +++ b/crates/sqlquack-syntax/src/expressions/tree/main/_on_grown_stack.rs @@ -0,0 +1,14 @@ +//! `on_grown_stack`: runs a step of a recursive walk on a fresh stack segment when the stack runs low. + +/// Runs `operation`, moving to a fresh stack segment first when the current stack is nearly used. +pub(in crate::expressions) fn on_grown_stack(operation: impl FnOnce() -> R) -> R { + #[cfg(feature = "stacker")] + { + stacker::maybe_grow(256 * 1024, 4 * 1024 * 1024, operation) + } + + #[cfg(not(feature = "stacker"))] + { + operation() + } +} diff --git a/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs b/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs index 75de2777f..adb3d149f 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/main/_serialize_on_grown_stack.rs @@ -3,6 +3,8 @@ use serde::Serialize; use serde::Serializer; +use crate::expressions::tree::main::_on_grown_stack::on_grown_stack; + /// Serializes `value`, moving to a fresh stack segment first when the current stack is nearly used. pub(in crate::expressions) fn serialize_on_grown_stack( value: &T, @@ -12,13 +14,5 @@ where T: Serialize + ?Sized, S: Serializer, { - #[cfg(feature = "stacker")] - { - stacker::maybe_grow(256 * 1024, 4 * 1024 * 1024, || value.serialize(serializer)) - } - - #[cfg(not(feature = "stacker"))] - { - value.serialize(serializer) - } + on_grown_stack(|| value.serialize(serializer)) } diff --git a/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs b/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs index 7c56a7206..130395b3d 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/main/mod.rs @@ -1,4 +1,5 @@ //! Main. +pub(in crate::expressions) mod _on_grown_stack; pub(in crate::expressions) mod _serialize_on_grown_stack; pub(in crate::expressions) mod _type_assertion_result; diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index fd204bb0d..84bb3bed7 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -51,6 +51,26 @@ fn given_a_long_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_re sql: || format!("SELECT 1{}", "+1".repeat(70_000)), expected_error: None, }, + AstDepthTestCase { + description: "a 50,000-term + chain as a call argument parses and serializes", + sql: || format!("SELECT f(1{}) FROM a", " + 1".repeat(50_000)), + expected_error: None, + }, + AstDepthTestCase { + description: "a 50,000-term + chain as a SUM argument parses and serializes", + sql: || format!("SELECT SUM(1{}) FROM a", " + 1".repeat(50_000)), + expected_error: None, + }, + AstDepthTestCase { + description: "a 50,000-term + chain as a COALESCE argument parses and serializes", + sql: || format!("SELECT COALESCE(1{}) FROM a", " + 1".repeat(50_000)), + expected_error: None, + }, + AstDepthTestCase { + description: "a 50,000-term + chain in ARRAY_CONSTRUCT parses and serializes", + sql: || format!("SELECT ARRAY_CONSTRUCT(1{}) FROM a", " + 1".repeat(50_000)), + expected_error: None, + }, AstDepthTestCase { description: "a 100,000-term + chain is flat, parses and serializes", sql: || format!("SELECT 1{}", "+1".repeat(100_000)), From 40101be61bdc3d87b812906232d2ff455fa5d040 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 19:42:54 +0000 Subject: [PATCH 04/22] fix: clone, compare and format deep syntax trees on a grown stack The derived Clone, PartialEq and Debug of Expression each keep a frame of several kilobytes per level in an unoptimised build, so on a 2 MiB thread equality overflowed at 150 nested CASE, UNION ALL, parenthesis, NOT or array levels, Debug at 300 and Clone at 500, all inside the 512-level AST depth guard, and on long operator chains. A new StackSafe derive generates the same three impls for Expression with each level run through on_grown_stack. Every recursive path passes through Expression, so this covers chains and nesting alike, and BinaryOp goes back to its derived impls. The test runs clone, equality, Debug, serialization and drop on a 2 MiB thread over 100,000-term chains and nesting just inside each depth guard. --- crates/sqlquack-syntax-derive/src/lib.rs | 11 +- .../src/stack_safe/main/expand_stack_safe.rs | 111 ++++++++++++++++++ .../src/stack_safe/main/mod.rs | 3 + .../src/stack_safe/mod.rs | 3 + .../src/expressions/operators/models.rs | 27 +---- .../src/expressions/tree/models.rs | 4 +- .../src/guard/tests/helpers.rs | 47 ++++++++ .../src/guard/tests/test_ast_guard.rs | 65 ++++++++++ .../src/guard/tests/test_types.rs | 7 ++ 9 files changed, 250 insertions(+), 28 deletions(-) create mode 100644 crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs create mode 100644 crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs create mode 100644 crates/sqlquack-syntax-derive/src/stack_safe/mod.rs diff --git a/crates/sqlquack-syntax-derive/src/lib.rs b/crates/sqlquack-syntax-derive/src/lib.rs index 82d500e5d..8b4ff08d6 100644 --- a/crates/sqlquack-syntax-derive/src/lib.rs +++ b/crates/sqlquack-syntax-derive/src/lib.rs @@ -1,15 +1,17 @@ -//! Derives exhaustive AST traversal: fields in order, containers walked, `#[ast(skip)]` skipped. +//! Derives exhaustive AST traversal, and stack-safe `Clone`, `PartialEq` and `Debug` for recursive enums. #![forbid(unsafe_code)] mod ast_node; mod constants; +mod stack_safe; use proc_macro::TokenStream; use syn::parse_macro_input; use syn::DeriveInput; use crate::ast_node::main::expand_ast_node::expand_ast_node; +use crate::stack_safe::main::expand_stack_safe::expand_stack_safe; #[proc_macro_derive(AstNode, attributes(ast))] /// Derive the expression-child visitors for an AST payload. @@ -17,3 +19,10 @@ pub fn derive_ast_node(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); expand_ast_node(&input).into() } + +#[proc_macro_derive(StackSafe)] +/// Derive `Clone`, `PartialEq` and `Debug` for a recursive enum, each on a grown stack when it runs low. +pub fn derive_stack_safe(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + expand_stack_safe(&input).into() +} diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs b/crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs new file mode 100644 index 000000000..509d83d30 --- /dev/null +++ b/crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs @@ -0,0 +1,111 @@ +//! `expand_stack_safe`. + +use quote::format_ident; +use quote::quote; +use syn::Data; +use syn::DeriveInput; +use syn::Fields; + +/// Derives `Clone`, `PartialEq` and `Debug` for an enum, each running one node on a grown stack. +pub(crate) fn expand_stack_safe(input: &DeriveInput) -> proc_macro2::TokenStream { + let name = &input.ident; + let Data::Enum(data) = &input.data else { + return quote!(compile_error!("StackSafe derives only for enums");); + }; + let mut clones: Vec = Vec::new(); + let mut comparisons: Vec = Vec::new(); + let mut formats: Vec = Vec::new(); + for variant in &data.variants { + let ident = &variant.ident; + let label = ident.to_string(); + let (pattern, other_pattern, bindings, others, rebuild, format) = match &variant.fields { + Fields::Unit => ( + quote!(Self::#ident), + quote!(Self::#ident), + Vec::new(), + Vec::new(), + quote!(Self::#ident), + quote!(formatter.write_str(#label)), + ), + Fields::Unnamed(fields) => { + let bindings: Vec<_> = (0..fields.unnamed.len()) + .map(|index| format_ident!("own_{index}")) + .collect(); + let others: Vec<_> = (0..fields.unnamed.len()) + .map(|index| format_ident!("other_{index}")) + .collect(); + ( + quote!(Self::#ident(#(#bindings),*)), + quote!(Self::#ident(#(#others),*)), + bindings.clone(), + others, + quote!(Self::#ident(#(::core::clone::Clone::clone(#bindings)),*)), + quote! {{ + let mut tuple = formatter.debug_tuple(#label); + #(tuple.field(#bindings);)* + tuple.finish() + }}, + ) + } + Fields::Named(fields) => { + let names: Vec<_> = fields + .named + .iter() + .filter_map(|field| field.ident.clone()) + .collect(); + let labels: Vec<_> = names.iter().map(ToString::to_string).collect(); + let bindings: Vec<_> = names + .iter() + .map(|field| format_ident!("own_{field}")) + .collect(); + let others: Vec<_> = names + .iter() + .map(|field| format_ident!("other_{field}")) + .collect(); + ( + quote!(Self::#ident { #(#names: #bindings),* }), + quote!(Self::#ident { #(#names: #others),* }), + bindings.clone(), + others, + quote!(Self::#ident { #(#names: ::core::clone::Clone::clone(#bindings)),* }), + quote! {{ + let mut fields = formatter.debug_struct(#label); + #(fields.field(#labels, #bindings);)* + fields.finish() + }}, + ) + } + }; + clones.push(quote!(#pattern => #rebuild)); + comparisons.push(quote!((#pattern, #other_pattern) => true #(&& #bindings == #others)*)); + formats.push(quote!(#pattern => #format)); + } + + quote! { + impl ::core::clone::Clone for #name { + fn clone(&self) -> Self { + crate::expressions::tree::main::_on_grown_stack::on_grown_stack(|| match self { + #(#clones,)* + }) + } + } + + impl ::core::cmp::PartialEq for #name { + #[allow(unreachable_patterns)] + fn eq(&self, other: &Self) -> bool { + crate::expressions::tree::main::_on_grown_stack::on_grown_stack(|| match (self, other) { + #(#comparisons,)* + _ => false, + }) + } + } + + impl ::core::fmt::Debug for #name { + fn fmt(&self, formatter: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { + crate::expressions::tree::main::_on_grown_stack::on_grown_stack(|| match self { + #(#formats,)* + }) + } + } + } +} diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs b/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs new file mode 100644 index 000000000..60bde10f4 --- /dev/null +++ b/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs @@ -0,0 +1,3 @@ +//! Main. + +pub(crate) mod expand_stack_safe; diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs b/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs new file mode 100644 index 000000000..ddcf80120 --- /dev/null +++ b/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs @@ -0,0 +1,3 @@ +//! The `StackSafe` derive. + +pub(crate) mod main; diff --git a/crates/sqlquack-syntax/src/expressions/operators/models.rs b/crates/sqlquack-syntax/src/expressions/operators/models.rs index 62fab949e..2f7e95cc4 100644 --- a/crates/sqlquack-syntax/src/expressions/operators/models.rs +++ b/crates/sqlquack-syntax/src/expressions/operators/models.rs @@ -7,7 +7,6 @@ use serde::Serialize; use crate::expressions::names::models::Identifier; use crate::expressions::names::models::TableRef; use crate::expressions::queries::models::Ordered; -use crate::expressions::tree::main::_on_grown_stack::on_grown_stack; use crate::expressions::tree::models::Expression; use crate::expressions::types::models::DataType; @@ -139,7 +138,7 @@ pub struct Case { } /// The payload of every binary operator variant: arithmetic, comparison, logical, bitwise. -#[derive(sqlquack_syntax_derive::AstNode, Debug, PartialEq, Serialize, Deserialize)] +#[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BinaryOp { #[serde( serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" @@ -168,30 +167,6 @@ pub struct BinaryOp { pub span: SourceSpan, } -/// Clones each operand on a grown stack when needed, so cloning a long chain cannot overflow. -impl Clone for BinaryOp { - fn clone(&self) -> Self { - let Self { - left, - right, - left_comments, - operator_comments, - trailing_comments, - inferred_type, - span, - } = self; - on_grown_stack(|| Self { - left: left.clone(), - right: right.clone(), - left_comments: left_comments.clone(), - operator_comments: operator_comments.clone(), - trailing_comments: trailing_comments.clone(), - inferred_type: inferred_type.clone(), - span: *span, - }) - } -} - impl BinaryOp { pub fn new(left: Expression, right: Expression) -> Self { Self { diff --git a/crates/sqlquack-syntax/src/expressions/tree/models.rs b/crates/sqlquack-syntax/src/expressions/tree/models.rs index f644be296..1d614bf6c 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/models.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/models.rs @@ -737,7 +737,9 @@ use crate::expressions::types::models::Struct; use crate::expressions::types::models::Tuple; /// Any SQL expression or statement, as one recursive AST node. -#[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive( + sqlquack_syntax_derive::AstNode, sqlquack_syntax_derive::StackSafe, Serialize, Deserialize, +)] #[serde(rename_all = "snake_case")] pub enum Expression { /// Structured Vertica syntax whose semantics are not shared by other dialects. diff --git a/crates/sqlquack-syntax/src/guard/tests/helpers.rs b/crates/sqlquack-syntax/src/guard/tests/helpers.rs index 325eb1ba3..3e4f6c7c5 100644 --- a/crates/sqlquack-syntax/src/guard/tests/helpers.rs +++ b/crates/sqlquack-syntax/src/guard/tests/helpers.rs @@ -91,3 +91,50 @@ pub(crate) fn constructor_levels(n: usize, open: &str) -> String { "[]".repeat(n - 61 * levels) ) } + +/// Default-guarded Snowflake trees with 100,000-term chains and nesting just inside each depth guard. +pub(crate) fn deep_trees() -> Vec { + let statements = [ + format!( + "SELECT f(1{}) FROM a WHERE a = 1{}", + " + 1".repeat(100_000), + " AND a = 1".repeat(50_000) + ), + format!( + "SELECT {}1{}", + "CASE WHEN TRUE THEN ".repeat(500), + " END".repeat(500) + ), + format!("SELECT 1{}", " UNION ALL SELECT 1".repeat(500)), + format!("SELECT {}1{}", "(".repeat(500), ")".repeat(500)), + format!("SELECT {}1{}", "(SELECT ".repeat(250), ")".repeat(250)), + format!("SELECT {}TRUE", "NOT ".repeat(500)), + format!("SELECT CAST(NULL AS INT{})", "[]".repeat(512)), + ]; + let mut trees = Vec::new(); + for sql in statements { + trees.extend( + crate::parse_statements( + &sql, + crate::Dialect::Snowflake, + &crate::ParseOptions::default(), + ) + .expect("the statement is inside every guard"), + ); + } + trees +} + +/// The number of bytes `value`'s `Debug` output takes, without keeping it. +pub(crate) fn debug_length(value: &impl std::fmt::Debug) -> usize { + struct Counter(usize); + impl std::fmt::Write for Counter { + fn write_str(&mut self, text: &str) -> std::fmt::Result { + self.0 += text.len(); + Ok(()) + } + } + let mut counter = Counter(0); + std::fmt::write(&mut counter, format_args!("{value:?}")).expect("formatting succeeds"); + counter.0 +} diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index 84bb3bed7..8077616ad 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -1,7 +1,10 @@ use crate::guard::tests::helpers::constructor_levels; +use crate::guard::tests::helpers::debug_length; +use crate::guard::tests::helpers::deep_trees; use crate::guard::tests::helpers::guarded_parse; use crate::guard::tests::helpers::guarded_parse_in; use crate::guard::tests::test_types::AstDepthTestCase; +use crate::guard::tests::test_types::TreeOperationTestCase; use crate::guard::tests::test_types::TypeDepthTestCase; use crate::guard::tests::test_types::TYPE_DEPTHS; @@ -71,6 +74,16 @@ fn given_a_long_term_chain_when_parsing_on_a_2_mb_thread_then_it_parses_or_is_re sql: || format!("SELECT ARRAY_CONSTRUCT(1{}) FROM a", " + 1".repeat(50_000)), expected_error: None, }, + AstDepthTestCase { + description: "a 70,000-cast :: chain inside a call is refused", + sql: || format!("SELECT f(1{}) FROM a", "::INT".repeat(70_000)), + expected_error: Some("E_GUARD_AST_DEPTH_EXCEEDED"), + }, + AstDepthTestCase { + description: "a 70,000-subscript chain inside a call is refused", + sql: || format!("SELECT f(v{}) FROM a", "[1]".repeat(70_000)), + expected_error: Some("E_GUARD_AST_DEPTH_EXCEEDED"), + }, AstDepthTestCase { description: "a 100,000-term + chain is flat, parses and serializes", sql: || format!("SELECT 1{}", "+1".repeat(100_000)), @@ -257,3 +270,55 @@ fn given_a_deeply_nested_type_when_parsing_on_a_2_mb_thread_then_512_levels_pars } } } + +#[test] +fn given_long_chains_and_guard_deep_nesting_on_a_2_mb_thread_when_cloning_comparing_formatting_serializing_or_dropping_then_none_overflows( +) { + let test_cases = [ + TreeOperationTestCase { + description: "clone", + operation: |tree| tree.clone().len() == tree.len(), + expected_completed: true, + }, + TreeOperationTestCase { + description: "equality with a clone, and inequality between different statements", + operation: |tree| { + let clone = tree.clone(); + clone == tree && clone[1..] != tree[..tree.len() - 1] + }, + expected_completed: true, + }, + TreeOperationTestCase { + description: "Debug formatting", + operation: |tree| debug_length(&tree) > 0, + expected_completed: true, + }, + TreeOperationTestCase { + description: "serialization", + operation: |tree| serde_json::to_writer(std::io::sink(), &tree).is_ok(), + expected_completed: true, + }, + TreeOperationTestCase { + description: "drop", + operation: |tree| { + drop(tree); + true + }, + expected_completed: true, + }, + ]; + for test_case in test_cases { + let operation = test_case.operation; + let completed = std::thread::Builder::new() + .stack_size(2 << 20) + .spawn(move || operation(deep_trees())) + .expect("the thread starts") + .join() + .expect("the thread finishes"); + assert_eq!( + completed, test_case.expected_completed, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-syntax/src/guard/tests/test_types.rs b/crates/sqlquack-syntax/src/guard/tests/test_types.rs index b49c13886..664a77430 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_types.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_types.rs @@ -50,3 +50,10 @@ pub(crate) struct TypeDepthTestCase { /// The nesting depths each [`TypeDepthTestCase`] is parsed at. pub(crate) const TYPE_DEPTHS: [usize; 4] = [512, 513, 20_000, 100_000]; + +pub(crate) struct TreeOperationTestCase { + pub(crate) description: &'static str, + /// The operation on the parsed tree; true when it ran to the end. + pub(crate) operation: fn(Vec) -> bool, + pub(crate) expected_completed: bool, +} From 783d26f6934840fb2aa159da92a787ff8bedeb0e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:13:56 +0000 Subject: [PATCH 05/22] Add sqlquack-fingerprint: canonical SQL tokens and change fingerprints One shared function for the SQL change fingerprints that SQLBuild and SQLClean both compute, so their hashes agree by construction rather than by test. canonical_tokens reads the significant tokens of SQL with the lossless tokenizer (generic, DuckDB, Snowflake, SQLite) or the native PostgreSQL scanner, and upper-cases unquoted reserved keywords and called names outside name positions. A word after '.', ':', '@', '$' or AS, a '__' placeholder call, the relation of an INTO or TABLE column list and a CTE name before its column list keep their case. query_fingerprint is the SHA-256 of those tokens, each framed as ':'. is_reserved_keyword exposes the per-dialect keyword lists. The digests match SQLBuild's recorded fingerprints, and every query of an 8,617-model corpus fingerprints identically on this revision and on 4d0c6bc. --- Cargo.lock | 9 + Cargo.toml | 2 + crates/sqlquack-fingerprint/Cargo.toml | 18 + .../src/fingerprint/_helpers/calls.rs | 67 ++++ .../src/fingerprint/_helpers/mod.rs | 5 + .../fingerprint/_helpers/reserved_keywords.rs | 21 + .../fingerprint/_helpers/significant_texts.rs | 41 ++ .../src/fingerprint/constants.rs | 16 + .../src/fingerprint/errors.rs | 25 ++ .../src/fingerprint/main/canonical_tokens.rs | 33 ++ .../fingerprint/main/is_reserved_keyword.rs | 17 + .../src/fingerprint/main/mod.rs | 5 + .../src/fingerprint/main/query_fingerprint.rs | 21 + .../src/fingerprint/mod.rs | 10 + .../src/fingerprint/models.rs | 32 ++ .../fingerprint/reserved_keywords/duckdb.txt | 77 ++++ .../fingerprint/reserved_keywords/generic.txt | 28 ++ .../reserved_keywords/postgres.txt | 80 ++++ .../reserved_keywords/snowflake.txt | 65 +++ .../src/fingerprint/tests.rs | 4 + .../src/fingerprint/tests/test_fingerprint.rs | 379 ++++++++++++++++++ .../src/fingerprint/tests/test_types.rs | 52 +++ crates/sqlquack-fingerprint/src/lib.rs | 11 + 23 files changed, 1018 insertions(+) create mode 100644 crates/sqlquack-fingerprint/Cargo.toml create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/_helpers/calls.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/_helpers/mod.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/_helpers/reserved_keywords.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/constants.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/errors.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/main/canonical_tokens.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/main/is_reserved_keyword.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/main/mod.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/main/query_fingerprint.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/mod.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/models.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/duckdb.txt create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/generic.txt create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/postgres.txt create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/snowflake.txt create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/tests.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/tests/test_fingerprint.rs create mode 100644 crates/sqlquack-fingerprint/src/fingerprint/tests/test_types.rs create mode 100644 crates/sqlquack-fingerprint/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 31cd36964..9f12eadc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,6 +1408,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sqlquack-fingerprint" +version = "0.2.0" +dependencies = [ + "sha2", + "sqlquack-syntax", + "sqlquack-syntax-postgres", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index fead36671..35fe529ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/sqlquack-polyglot-check", "crates/sqlquack-syntax-coverage", "crates/sqlquack-syntax-postgres", + "crates/sqlquack-fingerprint", "crates/sqlquack-python", ] exclude = ["fuzz"] @@ -57,6 +58,7 @@ sqlquack-compare = { path = "crates/sqlquack-compare" } sqlquack-core = { path = "crates/sqlquack-core" } sqlquack-duckdb = { path = "crates/sqlquack-duckdb", default-features = false } sqlquack-evidence = { path = "crates/sqlquack-evidence" } +sqlquack-fingerprint = { path = "crates/sqlquack-fingerprint" } sqlquack-policy = { path = "crates/sqlquack-policy" } sqlquack-rules = { path = "crates/sqlquack-rules" } sqlquack-semantics = { path = "crates/sqlquack-semantics" } diff --git a/crates/sqlquack-fingerprint/Cargo.toml b/crates/sqlquack-fingerprint/Cargo.toml new file mode 100644 index 000000000..2585ba684 --- /dev/null +++ b/crates/sqlquack-fingerprint/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sqlquack-fingerprint" +version.workspace = true +description = "Whitespace-, comment- and keyword-case-insensitive SQL change fingerprints over SQLQuack's tokenizers." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sha2.workspace = true +sqlquack-syntax.workspace = true +sqlquack-syntax-postgres.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/calls.rs b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/calls.rs new file mode 100644 index 000000000..6556056ca --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/calls.rs @@ -0,0 +1,67 @@ +//! Name positions, unquoted words and calls in an upper-cased token stream. + +use crate::fingerprint::constants::{ + ALIAS_KEYWORD, CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, + NAME_PREFIXES, PLACEHOLDER_PREFIX, +}; + +/// For every upper-cased token, whether it follows `.`, `:`, `@`, `$` or `AS`. +pub(crate) fn name_positions(uppers: &[String]) -> Vec { + let mut positions: Vec = Vec::with_capacity(uppers.len()); + let mut previous: Option<&str> = None; + for upper in uppers { + positions.push( + previous.is_some_and(|word| word == ALIAS_KEYWORD || NAME_PREFIXES.contains(&word)), + ); + previous = Some(upper.as_str()); + } + positions +} + +/// Whether `text` is an unquoted ASCII word whose letter case carries no meaning. +pub(crate) fn is_unquoted_word(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '$')) +} + +/// The index of the `)` closing each `(`, by the index of the `(`. +pub(crate) fn matching_closers(uppers: &[String]) -> Vec> { + let mut closers: Vec> = vec![None; uppers.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in uppers.iter().enumerate() { + if token == CALL_OPENER { + open.push(index); + } else if token == CALL_CLOSER + && let Some(opener) = open.pop() + { + closers[opener] = Some(index); + } + } + closers +} + +/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation. +pub(crate) fn calls(raw: &str, uppers: &[String], closers: &[Option], index: usize) -> bool { + if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER) + || raw.starts_with(PLACEHOLDER_PREFIX) + { + return false; + } + let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str()); + if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) { + return false; + } + let Some(closer) = closers[index + 1] else { + return true; + }; + !(uppers + .get(closer + 1) + .is_some_and(|next| next == ALIAS_KEYWORD) + && uppers + .get(closer + 2) + .is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str()))) +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/mod.rs b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/mod.rs new file mode 100644 index 000000000..6229ec967 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/mod.rs @@ -0,0 +1,5 @@ +//! Tokenizing, name positions and call detection behind the canonical token stream. + +pub(crate) mod calls; +pub(crate) mod reserved_keywords; +pub(crate) mod significant_texts; diff --git a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/reserved_keywords.rs b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/reserved_keywords.rs new file mode 100644 index 000000000..042f565c8 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/reserved_keywords.rs @@ -0,0 +1,21 @@ +//! Per-dialect reserved keyword sets, read once from the keyword lists. + +use std::collections::HashSet; +use std::sync::LazyLock; + +pub(crate) static DUCKDB: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt"))); +pub(crate) static POSTGRES: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt"))); +pub(crate) static SNOWFLAKE: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt"))); +pub(crate) static GENERIC: LazyLock> = + LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt"))); + +fn keyword_set(data: &str) -> HashSet { + data.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .map(str::to_ascii_uppercase) + .collect() +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs new file mode 100644 index 000000000..246297980 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs @@ -0,0 +1,41 @@ +//! The significant token texts of SQL under a dialect's tokenizer. + +use sqlquack_syntax::lossless::models::TokenKind; +use sqlquack_syntax::tokens::models::Dialect; +use sqlquack_syntax_postgres::scan::main::scan::scan; +use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind; + +use crate::fingerprint::errors::FingerprintError; +use crate::fingerprint::models::FingerprintDialect; + +/// The text of each significant token of `sql`, in order: no whitespace and no comments. +pub(crate) fn significant_texts( + sql: &str, + dialect: FingerprintDialect, +) -> Result, FingerprintError> { + let lossless: Dialect = match dialect { + FingerprintDialect::PostgreSQL => return postgres_texts(sql), + FingerprintDialect::Generic | FingerprintDialect::DuckDB => Dialect::DuckDB, + FingerprintDialect::Snowflake => Dialect::Snowflake, + FingerprintDialect::SQLite => Dialect::SQLite, + }; + Ok(sqlquack_syntax::tokenize(sql, lossless) + .map_err(|error| FingerprintError::Tokenize(error.to_string()))? + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql)) + .collect()) +} + +/// PostgreSQL's scanner tokens; a scanner fault fails the whole stream. +fn postgres_texts(sql: &str) -> Result, FingerprintError> { + scan(sql) + .into_iter() + .map(|token| match token.kind { + PostgresTokenKind::Fault(_) => Err(FingerprintError::Tokenize(token.value)), + _ => sql.get(token.start..token.end).ok_or_else(|| { + FingerprintError::Tokenize(format!("invalid token range at byte {}", token.start)) + }), + }) + .collect() +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/constants.rs b/crates/sqlquack-fingerprint/src/fingerprint/constants.rs new file mode 100644 index 000000000..24747f7bf --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/constants.rs @@ -0,0 +1,16 @@ +//! Tokens that decide whether a word folds. + +/// The token that follows a called function's name. +pub(crate) const CALL_OPENER: &str = "("; +/// The token that closes a call's arguments or a column list. +pub(crate) const CALL_CLOSER: &str = ")"; +/// Keywords after which a word followed by `(` names a relation and its column list. +pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"]; +/// The keyword between a CTE's column list and its body, and before an alias. +pub(crate) const ALIAS_KEYWORD: &str = "AS"; +/// The tokens that can follow `AS` after a CTE's column list. +pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"]; +/// Tokens after which any word, even a reserved one, is a name. +pub(crate) const NAME_PREFIXES: [&str; 4] = [".", ":", "@", "$"]; +/// The prefix of placeholder calls such as SQLBuild's `__ref` and `__source`. +pub(crate) const PLACEHOLDER_PREFIX: &str = "__"; diff --git a/crates/sqlquack-fingerprint/src/fingerprint/errors.rs b/crates/sqlquack-fingerprint/src/fingerprint/errors.rs new file mode 100644 index 000000000..7123da674 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/errors.rs @@ -0,0 +1,25 @@ +//! Fingerprint failures. + +use std::fmt::{self, Display}; + +/// SQL the dialect's tokenizer rejects, or a dialect name no tokenizer reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FingerprintError { + /// The tokenizer stopped; the message names why. + Tokenize(String), + /// No fingerprint dialect has this name. + UnknownDialect(String), +} + +impl Display for FingerprintError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Tokenize(message) => write!(formatter, "SQL tokenization failed: {message}"), + Self::UnknownDialect(name) => { + write!(formatter, "unknown SQL fingerprint dialect '{name}'") + } + } + } +} + +impl std::error::Error for FingerprintError {} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/main/canonical_tokens.rs b/crates/sqlquack-fingerprint/src/fingerprint/main/canonical_tokens.rs new file mode 100644 index 000000000..3511264e7 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/main/canonical_tokens.rs @@ -0,0 +1,33 @@ +//! `canonical_tokens`: the token stream a fingerprint hashes. + +use crate::fingerprint::_helpers::calls::{ + calls, is_unquoted_word, matching_closers, name_positions, +}; +use crate::fingerprint::_helpers::significant_texts::significant_texts; +use crate::fingerprint::errors::FingerprintError; +use crate::fingerprint::main::is_reserved_keyword::is_reserved_keyword; +use crate::fingerprint::models::FingerprintDialect; + +/// Significant token texts of `sql`, with unquoted reserved keywords and called names, outside name positions, upper-cased. +pub fn canonical_tokens( + sql: &str, + dialect: FingerprintDialect, +) -> Result, FingerprintError> { + let raws: Vec<&str> = significant_texts(sql, dialect)?; + let uppers: Vec = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect(); + let names: Vec = name_positions(&uppers); + let closers: Vec> = matching_closers(&uppers); + let mut texts: Vec = Vec::with_capacity(raws.len()); + for (index, raw) in raws.iter().enumerate() { + let folds = !names[index] + && is_unquoted_word(raw) + && (is_reserved_keyword(dialect, &uppers[index]) + || calls(raw, &uppers, &closers, index)); + texts.push(if folds { + uppers[index].clone() + } else { + (*raw).to_owned() + }); + } + Ok(texts) +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/main/is_reserved_keyword.rs b/crates/sqlquack-fingerprint/src/fingerprint/main/is_reserved_keyword.rs new file mode 100644 index 000000000..b8013f147 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/main/is_reserved_keyword.rs @@ -0,0 +1,17 @@ +//! `is_reserved_keyword`: whether a word can never be an unquoted identifier in a dialect. + +use std::collections::HashSet; + +use crate::fingerprint::_helpers::reserved_keywords::{DUCKDB, GENERIC, POSTGRES, SNOWFLAKE}; +use crate::fingerprint::models::FingerprintDialect; + +/// Whether the upper-cased `upper_word` is reserved in `dialect`; SQLite uses the generic list. +pub fn is_reserved_keyword(dialect: FingerprintDialect, upper_word: &str) -> bool { + let keywords: &HashSet = match dialect { + FingerprintDialect::DuckDB => &DUCKDB, + FingerprintDialect::PostgreSQL => &POSTGRES, + FingerprintDialect::Snowflake => &SNOWFLAKE, + FingerprintDialect::Generic | FingerprintDialect::SQLite => &GENERIC, + }; + keywords.contains(upper_word) +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/main/mod.rs b/crates/sqlquack-fingerprint/src/fingerprint/main/mod.rs new file mode 100644 index 000000000..e7f3234fe --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/main/mod.rs @@ -0,0 +1,5 @@ +//! Entry points. + +pub mod canonical_tokens; +pub mod is_reserved_keyword; +pub mod query_fingerprint; diff --git a/crates/sqlquack-fingerprint/src/fingerprint/main/query_fingerprint.rs b/crates/sqlquack-fingerprint/src/fingerprint/main/query_fingerprint.rs new file mode 100644 index 000000000..9c1c163e6 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/main/query_fingerprint.rs @@ -0,0 +1,21 @@ +//! `query_fingerprint`: a whitespace-, comment- and keyword-case-insensitive SQL change fingerprint. + +use sha2::{Digest, Sha256}; + +use crate::fingerprint::errors::FingerprintError; +use crate::fingerprint::main::canonical_tokens::canonical_tokens; +use crate::fingerprint::models::FingerprintDialect; + +/// The SHA-256 hex digest of `sql`'s canonical tokens, each framed as `:`. +pub fn query_fingerprint( + sql: &str, + dialect: FingerprintDialect, +) -> Result { + let mut digest = Sha256::new(); + for text in canonical_tokens(sql, dialect)? { + digest.update(text.len().to_string().as_bytes()); + digest.update(b":"); + digest.update(text.as_bytes()); + } + Ok(format!("{:x}", digest.finalize())) +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/mod.rs b/crates/sqlquack-fingerprint/src/fingerprint/mod.rs new file mode 100644 index 000000000..a726a7f91 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/mod.rs @@ -0,0 +1,10 @@ +//! The canonical token stream of SQL and its SHA-256 fingerprint. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-fingerprint/src/fingerprint/models.rs b/crates/sqlquack-fingerprint/src/fingerprint/models.rs new file mode 100644 index 000000000..11650ad53 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/models.rs @@ -0,0 +1,32 @@ +//! The dialects a fingerprint reads SQL in. + +use crate::fingerprint::errors::FingerprintError; + +/// A dialect SQLQuack tokenizes for fingerprints; `Generic` reads DuckDB's tokens with a smaller keyword list. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FingerprintDialect { + /// No named dialect. + Generic, + /// DuckDB. + DuckDB, + /// PostgreSQL, through its native scanner. + PostgreSQL, + /// Snowflake. + Snowflake, + /// SQLite. + SQLite, +} + +impl FingerprintDialect { + /// The dialect named `name`: `generic`, `duckdb`, `postgres`, `postgresql`, `snowflake` or `sqlite`. + pub fn from_name(name: &str) -> Result { + match name { + "generic" => Ok(Self::Generic), + "duckdb" => Ok(Self::DuckDB), + "postgres" | "postgresql" => Ok(Self::PostgreSQL), + "snowflake" => Ok(Self::Snowflake), + "sqlite" => Ok(Self::SQLite), + _ => Err(FingerprintError::UnknownDialect(name.to_owned())), + } + } +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/duckdb.txt b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/duckdb.txt new file mode 100644 index 000000000..4374c4e53 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/duckdb.txt @@ -0,0 +1,77 @@ +# DuckDB reserved keywords: SELECT keyword_name FROM duckdb_keywords() +# WHERE keyword_category = 'reserved' (DuckDB 1.5.2). +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +default +deferrable +desc +describe +distinct +do +else +end +except +false +fetch +for +foreign +from +group +having +in +initially +intersect +into +lambda +lateral +leading +limit +not +null +offset +on +only +or +order +pivot +pivot_longer +pivot_wider +placing +primary +qualify +references +returning +select +show +some +summarize +symmetric +table +then +to +trailing +true +union +unique +unpivot +using +variadic +when +where +window +with diff --git a/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/generic.txt b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/generic.txt new file mode 100644 index 000000000..05f8ee42c --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/generic.txt @@ -0,0 +1,28 @@ +# Generic dialect: keywords reserved in every dialect file above except Databricks +# (DuckDB, PostgreSQL, BigQuery, Snowflake and T-SQL). +all +and +any +as +create +distinct +else +for +from +group +having +in +intersect +into +not +null +on +or +order +select +some +then +to +union +where +with diff --git a/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/postgres.txt b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/postgres.txt new file mode 100644 index 000000000..7a57911cb --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/postgres.txt @@ -0,0 +1,80 @@ +# PostgreSQL reserved keywords (non-function): RESERVED_KEYWORD entries of +# src/include/parser/kwlist.h (REL_17_STABLE); Appendix C 'reserved'. +all +analyse +analyze +and +any +array +as +asc +asymmetric +both +case +cast +check +collate +column +constraint +create +current_catalog +current_date +current_role +current_time +current_timestamp +current_user +default +deferrable +desc +distinct +do +else +end +except +false +fetch +for +foreign +from +grant +group +having +in +initially +intersect +into +lateral +leading +limit +localtime +localtimestamp +not +null +offset +on +only +or +order +placing +primary +references +returning +select +session_user +some +symmetric +system_user +table +then +to +trailing +true +union +unique +user +using +variadic +when +where +window +with diff --git a/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/snowflake.txt b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/snowflake.txt new file mode 100644 index 000000000..511b64668 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/reserved_keywords/snowflake.txt @@ -0,0 +1,65 @@ +# Snowflake reserved keywords: entries marked 'Reserved by ANSI', 'Reserved by Snowflake' +# or 'Reserved by Snowflake and others' in https://docs.snowflake.com/en/sql-reference/reserved-keywords. +# Limited keywords (usable as identifiers in some positions) are excluded. +all +alter +and +any +as +between +by +check +column +connect +create +current +delete +distinct +drop +else +exists +following +for +from +grant +group +having +ilike +in +increment +insert +intersect +into +is +like +minus +not +null +of +on +or +order +qualify +regexp +revoke +rlike +row +rows +sample +select +set +some +start +table +tablesample +then +to +trigger +union +unique +update +values +whenever +where +window +with diff --git a/crates/sqlquack-fingerprint/src/fingerprint/tests.rs b/crates/sqlquack-fingerprint/src/fingerprint/tests.rs new file mode 100644 index 000000000..b432653ad --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_fingerprint.rs"] +mod test_fingerprint; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-fingerprint/src/fingerprint/tests/test_fingerprint.rs b/crates/sqlquack-fingerprint/src/fingerprint/tests/test_fingerprint.rs new file mode 100644 index 000000000..f20d5defa --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/tests/test_fingerprint.rs @@ -0,0 +1,379 @@ +use crate::fingerprint::errors::FingerprintError; +use crate::fingerprint::main::canonical_tokens::canonical_tokens; +use crate::fingerprint::main::query_fingerprint::query_fingerprint; +use crate::fingerprint::models::FingerprintDialect; +use crate::fingerprint::tests::test_types::{ + DialectNameTestCase, FailureTestCase, GoldenTestCase, NestingTestCase, StabilityTestCase, + TokensTestCase, +}; + +/// Nesting depth of the pathological inputs. +const NESTING: usize = 32_000; +/// Room for linear work at that depth in a debug build, but not for a quadratic path. +const NESTING_LIMIT: std::time::Duration = std::time::Duration::from_secs(2); + +#[test] +fn given_recorded_queries_when_fingerprinted_then_digests_never_change() { + let test_cases = [ + GoldenTestCase { + description: "generic token stream", + dialect: FingerprintDialect::Generic, + sql: "SELECT id, name FROM orders", + expected_digest: "264bcb1f81ce5ba25394a052016edc265aefba85c0698f616d3037f9d872d2bd", + }, + GoldenTestCase { + description: "DuckDB dollar quote and cast", + dialect: FingerprintDialect::DuckDB, + sql: "SELECT o.id, $$Customer's note$$ AS note, amount::DECIMAL(10, 2) AS amount FROM orders AS o", + expected_digest: "07786643d9593498ac2b2394e6801cb578b23ba0f9a0e70dbcdaf0d3a3cbf9db", + }, + GoldenTestCase { + description: "PostgreSQL tagged dollar quote", + dialect: FingerprintDialect::PostgreSQL, + sql: "SELECT o.id, $tag$it's$tag$ AS note, amount::NUMERIC AS amount FROM orders AS o", + expected_digest: "8db865c6f239b990c917754df176bd3c9b2c0c322b9af1437298ec98b67241f9", + }, + GoldenTestCase { + description: "Snowflake path access and QUALIFY", + dialect: FingerprintDialect::Snowflake, + sql: "SELECT o.id, payload:customer.name::STRING AS customer FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.id ORDER BY o.id) = 1", + expected_digest: "c75f7b5931a18e1a074ffa7ab113c256fc838285af75a3f9feb62d61cc76d94d", + }, + GoldenTestCase { + description: "DuckDB lower-case keywords and SQLBuild placeholder", + dialect: FingerprintDialect::DuckDB, + sql: "select o.id from __ref(\"orders\") as o where o.id > 0", + expected_digest: "fda88b9b0f26486bdd25b8e3bab5a982fbb387901ab229091b17772d044aebd0", + }, + ]; + for test_case in test_cases { + assert_eq!( + query_fingerprint(test_case.sql, test_case.dialect).as_deref(), + Ok(test_case.expected_digest), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_two_queries_when_fingerprinted_then_only_meaningful_edits_change_the_digest() { + let test_cases = [ + StabilityTestCase { + description: "whitespace and line breaks are ignored", + dialect: FingerprintDialect::DuckDB, + first: "SELECT id FROM orders", + second: "select\n id\nfrom orders", + expected_same: true, + }, + StabilityTestCase { + description: "comments are ignored", + dialect: FingerprintDialect::Snowflake, + first: "SELECT id -- order key\nFROM orders /* all */", + second: "SELECT id FROM orders", + expected_same: true, + }, + StabilityTestCase { + description: "keyword and built-in function case is ignored", + dialect: FingerprintDialect::DuckDB, + first: "SELECT COUNT(id) FROM orders WHERE id > 0 LIMIT 1", + second: "select count(id) from orders where id > 0 limit 1", + expected_same: true, + }, + StabilityTestCase { + description: "unquoted identifier case is significant", + dialect: FingerprintDialect::DuckDB, + first: "SELECT Id FROM Orders", + second: "SELECT id FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "alias case is significant", + dialect: FingerprintDialect::DuckDB, + first: "SELECT 1 AS Foo", + second: "SELECT 1 AS foo", + expected_same: false, + }, + StabilityTestCase { + description: "Snowflake path key case is significant", + dialect: FingerprintDialect::Snowflake, + first: "SELECT payload:customerId FROM orders", + second: "SELECT payload:customerid FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "unquoted call name case is ignored", + dialect: FingerprintDialect::DuckDB, + first: "SELECT MyFunc(id) FROM orders", + second: "SELECT myfunc(id) FROM orders", + expected_same: true, + }, + StabilityTestCase { + description: "quoted call name case is significant", + dialect: FingerprintDialect::Snowflake, + first: "SELECT \"MyFunc\"(id) FROM orders", + second: "SELECT \"myfunc\"(id) FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "PostgreSQL multi-character operator is one token", + dialect: FingerprintDialect::PostgreSQL, + first: "SELECT a |>> b FROM orders", + second: "SELECT a |> > b FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "CTE column-list name case is significant", + dialect: FingerprintDialect::DuckDB, + first: "WITH Tree(id) AS (SELECT 1) SELECT id FROM orders", + second: "WITH tree(id) AS (SELECT 1) SELECT id FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "SQLBuild placeholder case is significant", + dialect: FingerprintDialect::DuckDB, + first: "SELECT id FROM __ref(\"orders\")", + second: "SELECT id FROM __REF(\"orders\")", + expected_same: false, + }, + StabilityTestCase { + description: "PostgreSQL comments, layout and keyword case are ignored", + dialect: FingerprintDialect::PostgreSQL, + first: "SELECT o.id -- key\nFROM orders AS o WHERE o.id > 0", + second: "select o.id from orders as o /* all */ where o.id > 0", + expected_same: true, + }, + StabilityTestCase { + description: "PostgreSQL dollar-quoted body is significant", + dialect: FingerprintDialect::PostgreSQL, + first: "SELECT $tag$it's$tag$ AS note", + second: "SELECT $tag$it s$tag$ AS note", + expected_same: false, + }, + StabilityTestCase { + description: "SQLite comments and keyword case are ignored", + dialect: FingerprintDialect::SQLite, + first: "SELECT id FROM orders -- trailing", + second: "select id\nfrom orders", + expected_same: true, + }, + StabilityTestCase { + description: "SQLite identifier case is significant", + dialect: FingerprintDialect::SQLite, + first: "SELECT Id FROM orders", + second: "SELECT id FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "Snowflake double-slash comments are ignored", + dialect: FingerprintDialect::Snowflake, + first: "SELECT id // order key\nFROM orders", + second: "SELECT id FROM orders", + expected_same: true, + }, + StabilityTestCase { + description: "generic keyword case is ignored and identifier case is not", + dialect: FingerprintDialect::Generic, + first: "select Id from orders", + second: "SELECT id FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "string literal whitespace is significant", + dialect: FingerprintDialect::DuckDB, + first: "SELECT 'a b' AS code", + second: "SELECT 'a b' AS code", + expected_same: false, + }, + StabilityTestCase { + description: "quoted identifier case is significant", + dialect: FingerprintDialect::Snowflake, + first: "SELECT \"Id\" FROM orders", + second: "SELECT \"ID\" FROM orders", + expected_same: false, + }, + StabilityTestCase { + description: "number spelling is significant", + dialect: FingerprintDialect::DuckDB, + first: "SELECT 1.0 AS amount", + second: "SELECT 1.00 AS amount", + expected_same: false, + }, + StabilityTestCase { + description: "different queries produce different hashes", + dialect: FingerprintDialect::Generic, + first: "SELECT id FROM orders", + second: "SELECT id FROM customers", + expected_same: false, + }, + ]; + for test_case in test_cases { + let first = query_fingerprint(test_case.first, test_case.dialect); + let second = query_fingerprint(test_case.second, test_case.dialect); + assert!(first.is_ok() && second.is_ok(), "{}", test_case.description); + assert_eq!( + first == second, + test_case.expected_same, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_canonicalized_then_only_keywords_and_called_names_fold() { + let test_cases = [ + TokensTestCase { + description: "keywords and a called function fold; identifiers and aliases keep their case", + dialect: FingerprintDialect::DuckDB, + sql: "select count(Id) as Total from Orders", + expected_tokens: &[ + "SELECT", "COUNT", "(", "Id", ")", "AS", "Total", "FROM", "Orders", + ], + }, + TokensTestCase { + description: "a reserved word after a dot or AS is a name", + dialect: FingerprintDialect::DuckDB, + sql: "select o.select as from_ from o", + expected_tokens: &["SELECT", "o", ".", "select", "AS", "from_", "FROM", "o"], + }, + TokensTestCase { + description: "placeholder calls and INTO column lists keep their case; INSERT is unreserved in DuckDB", + dialect: FingerprintDialect::DuckDB, + sql: "insert into Orders(id) select id from __ref(\"x\")", + expected_tokens: &[ + "insert", "INTO", "Orders", "(", "id", ")", "SELECT", "id", "FROM", "__ref", "(", + "\"x\"", ")", + ], + }, + TokensTestCase { + description: "a CTE column list is not a call, but an unclosed call still is", + dialect: FingerprintDialect::DuckDB, + sql: "with Tree(a) as (select f(1)) select g(", + expected_tokens: &[ + "WITH", "Tree", "(", "a", ")", "AS", "(", "SELECT", "F", "(", "1", ")", ")", + "SELECT", "G", "(", + ], + }, + TokensTestCase { + description: "PostgreSQL's scanner keeps a multi-character operator and a dollar body whole", + dialect: FingerprintDialect::PostgreSQL, + sql: "select a |>> $t$Body$t$ -- note", + expected_tokens: &["SELECT", "a", "|>>", "$t$Body$t$"], + }, + TokensTestCase { + description: "a word reserved only in Snowflake folds only there", + dialect: FingerprintDialect::Generic, + sql: "select qualify from t", + expected_tokens: &["SELECT", "qualify", "FROM", "t"], + }, + ]; + for test_case in test_cases { + let tokens = canonical_tokens(test_case.sql, test_case.dialect); + assert_eq!( + tokens + .as_deref() + .map(|tokens| tokens.iter().map(String::as_str).collect::>()), + Ok(test_case.expected_tokens.to_vec()), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_untokenizable_sql_when_fingerprinted_then_the_tokenizer_message_is_reported() { + let test_cases = [ + FailureTestCase { + description: "an unterminated string under the lossless tokenizer", + dialect: FingerprintDialect::DuckDB, + sql: "SELECT 'open FROM orders", + expected_fragment: "Unterminated string", + }, + FailureTestCase { + description: "an unterminated string under the PostgreSQL scanner", + dialect: FingerprintDialect::PostgreSQL, + sql: "SELECT 'open FROM orders", + expected_fragment: "SQL tokenization failed: unterminated", + }, + ]; + for test_case in test_cases { + let message: String = query_fingerprint(test_case.sql, test_case.dialect) + .err() + .map(|error: FingerprintError| error.to_string()) + .unwrap_or_default(); + assert!( + message.contains(test_case.expected_fragment), + "{}: {message}", + test_case.description + ); + } +} + +#[test] +fn given_pathological_nesting_when_fingerprinted_then_time_stays_linear() { + let test_cases = [ + NestingTestCase { + description: "deeply nested parentheses", + dialect: FingerprintDialect::DuckDB, + sql: format!("SELECT 1 + {}1{}", "(".repeat(NESTING), ")".repeat(NESTING)), + expected_within_limit: true, + }, + NestingTestCase { + description: "deeply nested calls", + dialect: FingerprintDialect::PostgreSQL, + sql: format!("SELECT {}1{}", "f(".repeat(NESTING), ")".repeat(NESTING)), + expected_within_limit: true, + }, + NestingTestCase { + description: "unclosed calls", + dialect: FingerprintDialect::Snowflake, + sql: format!("SELECT {}", "f(".repeat(NESTING)), + expected_within_limit: true, + }, + ]; + for test_case in test_cases { + let started = std::time::Instant::now(); + let _ = query_fingerprint(&test_case.sql, test_case.dialect); + assert_eq!( + started.elapsed() < NESTING_LIMIT, + test_case.expected_within_limit, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dialect_names_when_resolved_then_both_postgres_spellings_agree() { + let test_cases = [ + DialectNameTestCase { + description: "the short PostgreSQL name", + name: "postgres", + expected_dialect: Ok(FingerprintDialect::PostgreSQL), + }, + DialectNameTestCase { + description: "the long PostgreSQL name", + name: "postgresql", + expected_dialect: Ok(FingerprintDialect::PostgreSQL), + }, + DialectNameTestCase { + description: "no named dialect", + name: "generic", + expected_dialect: Ok(FingerprintDialect::Generic), + }, + DialectNameTestCase { + description: "a dialect SQLQuack does not tokenize", + name: "tsql", + expected_dialect: Err(FingerprintError::UnknownDialect("tsql".into())), + }, + ]; + for test_case in test_cases { + assert_eq!( + FingerprintDialect::from_name(test_case.name), + test_case.expected_dialect, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-fingerprint/src/fingerprint/tests/test_types.rs b/crates/sqlquack-fingerprint/src/fingerprint/tests/test_types.rs new file mode 100644 index 000000000..afb2655a0 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/fingerprint/tests/test_types.rs @@ -0,0 +1,52 @@ +//! Test case types for fingerprints. + +use crate::fingerprint::errors::FingerprintError; +use crate::fingerprint::models::FingerprintDialect; + +/// SQL whose fingerprint must stay the recorded digest. +pub(crate) struct GoldenTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: FingerprintDialect, + pub(crate) sql: &'static str, + pub(crate) expected_digest: &'static str, +} + +/// Two queries and whether they must share a fingerprint. +pub(crate) struct StabilityTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: FingerprintDialect, + pub(crate) first: &'static str, + pub(crate) second: &'static str, + pub(crate) expected_same: bool, +} + +/// SQL and its exact canonical token stream. +pub(crate) struct TokensTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: FingerprintDialect, + pub(crate) sql: &'static str, + pub(crate) expected_tokens: &'static [&'static str], +} + +/// SQL that must fail with a message containing `expected_fragment`. +pub(crate) struct FailureTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: FingerprintDialect, + pub(crate) sql: &'static str, + pub(crate) expected_fragment: &'static str, +} + +/// Pathological SQL and whether its fingerprint finishes within the nesting limit. +pub(crate) struct NestingTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: FingerprintDialect, + pub(crate) sql: String, + pub(crate) expected_within_limit: bool, +} + +/// A dialect name and the dialect it resolves to. +pub(crate) struct DialectNameTestCase { + pub(crate) description: &'static str, + pub(crate) name: &'static str, + pub(crate) expected_dialect: Result, +} diff --git a/crates/sqlquack-fingerprint/src/lib.rs b/crates/sqlquack-fingerprint/src/lib.rs new file mode 100644 index 000000000..f51d7eaa2 --- /dev/null +++ b/crates/sqlquack-fingerprint/src/lib.rs @@ -0,0 +1,11 @@ +//! Whitespace-, comment- and keyword-case-insensitive SQL change fingerprints over SQLQuack's tokenizers. + +#![forbid(unsafe_code)] + +pub mod fingerprint; + +pub use crate::fingerprint::errors::FingerprintError; +pub use crate::fingerprint::main::canonical_tokens::canonical_tokens; +pub use crate::fingerprint::main::is_reserved_keyword::is_reserved_keyword; +pub use crate::fingerprint::main::query_fingerprint::query_fingerprint; +pub use crate::fingerprint::models::FingerprintDialect; From 1b1deed0b4c46f5466ef725015f89b64afd95562 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:14:45 +0000 Subject: [PATCH 06/22] Postgres parser refuses a statement tree deeper than consumers can walk Chains nested in another chain's first operand escaped the per-chain and per-level counts, so a statement could build a tree tens of thousands of levels deep, which aborts recursive serialisation. Each statement's tree depth is now checked after parsing and refused past 8,000 levels with the stack depth error. --- .../src/parse/_helpers/tree/cursor.rs | 13 +++++++- .../src/parse/main/parse.rs | 1 + .../src/parse/tests/test_depth.rs | 30 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs index 2bd41af15..b1c4afc92 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; -use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Identifier; +use sqlquack_syntax::{Expression, ExpressionWalk}; use crate::keywords::models::KeywordCategory; use crate::parse::constants::{EXPRESSION_KEYWORDS, QUERY_STARTS}; @@ -19,6 +19,9 @@ const MAX_DEPTH: usize = 300; /// The longest flat chain of operators other than AND and OR, or of indirections. const MAX_CHAIN: usize = 1000; +/// The deepest statement tree; a release build serialises one this deep on a 2 MiB thread. +const MAX_TREE_DEPTH: usize = 8000; + /// PostgreSQL's message for 54001, which recursion and chains past their limits both raise. const STACK_DEPTH_EXCEEDED: &str = "stack depth limit exceeded"; @@ -382,6 +385,14 @@ impl<'sql> Parser<'sql> { } } + /// Refuses `statement` when its tree nests deeper than `MAX_TREE_DEPTH`, as 54001. + pub(crate) fn within_tree_depth(&mut self, statement: &Expression) -> Parsing<()> { + match statement.tree_depth() > MAX_TREE_DEPTH { + true => Err(self.refuse_depth()), + false => Ok(()), + } + } + /// A flat chain's link count after one more link, refusing the statement past `MAX_CHAIN`. pub(crate) fn link(&mut self, links: usize) -> Parsing { match links < MAX_CHAIN { diff --git a/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs b/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs index b6ded1df7..bea5729b7 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/main/parse.rs @@ -24,6 +24,7 @@ pub fn parse(sql: &str) -> Result { parser.opaque = false; let statement: Parsing = parser.statement(); let statement: Expression = parser.unless_too_deep(statement)?; + parser.within_tree_depth(&statement)?; if !(parser.peek().is_none() || parser.is_char(';')) { return Err(parser.error()); } diff --git a/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs index 3907e7780..d6490faec 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs @@ -350,6 +350,16 @@ fn given_deep_nesting_when_parsing_on_a_small_stack_then_the_depth_limit_refuses levels: 100000, expected_error: Some(("stack depth limit exceeded", false)), }, + NestingTestCase { + description: "an OR chain past the tree depth", + prefix: "SELECT true", + open: "", + innermost: "", + close: " OR true", + read: |text| parse(text).err(), + levels: 20000, + expected_error: Some(("stack depth limit exceeded", false)), + }, NestingTestCase { description: "AND chains are not counted", prefix: "SELECT true", @@ -380,6 +390,26 @@ fn given_deep_nesting_when_parsing_on_a_small_stack_then_the_depth_limit_refuses levels: 2000, expected_error: Some(("stack depth limit exceeded", false)), }, + NestingTestCase { + description: "cast chains nested in each other's first operand, past the tree depth", + prefix: "SELECT ", + open: "(", + innermost: "x", + close: ")::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int::int", + read: |text| parse(text).err(), + levels: 299, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "cast chains nested in each other's first operand, within the tree depth", + prefix: "SELECT ", + open: "(", + innermost: "x", + close: ")::int::int::int::int::int::int::int::int::int::int", + read: |text| parse(text).err(), + levels: 150, + expected_error: None, + }, NestingTestCase { description: "parenthesized type modifiers in a type string well inside the limit", prefix: "", From a8fdf9f267fe2f1c1c54bc61267a3d6dafdca316 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:18:09 +0000 Subject: [PATCH 07/22] Postgres parser README records the tree depth limit and the band the engine accepts --- crates/sqlquack-syntax-postgres/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/sqlquack-syntax-postgres/README.md b/crates/sqlquack-syntax-postgres/README.md index 86ddd5e23..3321ca84a 100644 --- a/crates/sqlquack-syntax-postgres/README.md +++ b/crates/sqlquack-syntax-postgres/README.md @@ -20,8 +20,14 @@ Nesting is bounded so that no input exhausts the caller's stack or takes exponen one reading passes the limit, every other reading fails at once and the statement is refused. PostgreSQL's own limit depends on its stack size, so it accepts some deeper texts. - A flat chain of operators (other than `AND` and `OR`), casts or indirections is refused with - the same error past 1,000 links. `AND`, `OR` and set operations are not counted; their trees are - as deep as the chain, so a consumer that walks them recursively must bound the depth itself. + the same error past 1,000 links. `AND`, `OR` and set operations are not counted. +- Each statement's tree, as `tree_depth` measures it, is refused with the same error past 8,000 + levels. This covers chains nested in another chain's first operand, which the counts above miss, + and long `AND`, `OR` and set-operation chains. A release build serialises such a tree on a + 2 MiB thread. +- Local parser limit, tree depth: PostgreSQL 18.6 accepts nested casts to about 13,200 levels + (132 levels of 99 casts each) before its own 54001, so a statement 8,000 to 13,200 levels deep is + refused here and accepted by the engine. - A parenthesized query, and `JSON_OBJECT`'s arguments, are parsed once from each token and the outcome replayed, so alternatives that each read the same nested text stay linear. From f0cd7147933752c1d392d4b94242304898d34114 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:24:56 +0000 Subject: [PATCH 08/22] Match SQLBuild's message for a scanner token outside the SQL --- .../src/fingerprint/_helpers/significant_texts.rs | 6 +++--- crates/sqlquack-fingerprint/src/fingerprint/errors.rs | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs index 246297980..0bf99f06d 100644 --- a/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs +++ b/crates/sqlquack-fingerprint/src/fingerprint/_helpers/significant_texts.rs @@ -33,9 +33,9 @@ fn postgres_texts(sql: &str) -> Result, FingerprintError> { .into_iter() .map(|token| match token.kind { PostgresTokenKind::Fault(_) => Err(FingerprintError::Tokenize(token.value)), - _ => sql.get(token.start..token.end).ok_or_else(|| { - FingerprintError::Tokenize(format!("invalid token range at byte {}", token.start)) - }), + _ => sql + .get(token.start..token.end) + .ok_or(FingerprintError::TokenRange(token.start)), }) .collect() } diff --git a/crates/sqlquack-fingerprint/src/fingerprint/errors.rs b/crates/sqlquack-fingerprint/src/fingerprint/errors.rs index 7123da674..08876803b 100644 --- a/crates/sqlquack-fingerprint/src/fingerprint/errors.rs +++ b/crates/sqlquack-fingerprint/src/fingerprint/errors.rs @@ -7,6 +7,8 @@ use std::fmt::{self, Display}; pub enum FingerprintError { /// The tokenizer stopped; the message names why. Tokenize(String), + /// The PostgreSQL scanner returned a token outside the SQL, at this byte offset. + TokenRange(usize), /// No fingerprint dialect has this name. UnknownDialect(String), } @@ -15,6 +17,7 @@ impl Display for FingerprintError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Tokenize(message) => write!(formatter, "SQL tokenization failed: {message}"), + Self::TokenRange(start) => write!(formatter, "SQL tokenization failed at byte {start}"), Self::UnknownDialect(name) => { write!(formatter, "unknown SQL fingerprint dialect '{name}'") } From d538017e11c6183756d8b0903e67a75e75928cd1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:35:12 +0000 Subject: [PATCH 09/22] fix: serialize deep syntax trees on a grown stack Expression's Serialize was still serde's derive, and only BinaryOp operands and DataType moved to a fresh stack segment. Any other nesting, such as casts, CASE or parentheses, overflowed a 2 MiB debug thread at a few hundred levels once a parser accepts it. The StackSafe derive now also generates Serialize, in serde's externally tagged snake_case form, with each level run through on_grown_stack. It refuses to compile for other serde attributes. Expression drops serde's Serialize derive, and BinaryOp drops its per-field serialize_with. The derive moves into the ast_node domain next to AstNode. The serialized JSON and Debug output of 63,375 corpus parses are byte-identical to before. The test adds 8,000-level casts, CASE and parentheses, parsed with raised guards, to the trees it clones, compares, formats, serializes and drops on a 2 MiB thread. --- .../main/expand_stack_safe.rs | 67 ++++++++++++++++++- .../src/ast_node/main/mod.rs | 1 + .../src/ast_node/mod.rs | 2 +- .../sqlquack-syntax-derive/src/constants.rs | 3 + crates/sqlquack-syntax-derive/src/lib.rs | 7 +- .../src/stack_safe/main/mod.rs | 3 - .../src/stack_safe/mod.rs | 3 - .../src/expressions/operators/models.rs | 6 -- .../src/expressions/tree/models.rs | 5 +- .../src/guard/tests/helpers.rs | 37 +++++++--- .../src/guard/tests/test_ast_guard.rs | 10 ++- 11 files changed, 110 insertions(+), 34 deletions(-) rename crates/sqlquack-syntax-derive/src/{stack_safe => ast_node}/main/expand_stack_safe.rs (61%) delete mode 100644 crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs delete mode 100644 crates/sqlquack-syntax-derive/src/stack_safe/mod.rs diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs b/crates/sqlquack-syntax-derive/src/ast_node/main/expand_stack_safe.rs similarity index 61% rename from crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs rename to crates/sqlquack-syntax-derive/src/ast_node/main/expand_stack_safe.rs index 509d83d30..8e1a78ac6 100644 --- a/crates/sqlquack-syntax-derive/src/stack_safe/main/expand_stack_safe.rs +++ b/crates/sqlquack-syntax-derive/src/ast_node/main/expand_stack_safe.rs @@ -6,18 +6,38 @@ use syn::Data; use syn::DeriveInput; use syn::Fields; -/// Derives `Clone`, `PartialEq` and `Debug` for an enum, each running one node on a grown stack. +use crate::ast_node::_helpers::naming::serde_snake_case; +use crate::constants::SNAKE_CASE; + +/// Derives `Clone`, `PartialEq`, `Debug` and `Serialize` for an enum, each running one node on a grown stack. pub(crate) fn expand_stack_safe(input: &DeriveInput) -> proc_macro2::TokenStream { let name = &input.ident; let Data::Enum(data) = &input.data else { return quote!(compile_error!("StackSafe derives only for enums");); }; + if !has_only_snake_case_serde_attributes(input) { + return quote!(compile_error!( + "StackSafe serializes like serde's `rename_all = \"snake_case\"` with no other serde attribute" + );); + } + let type_label = name.to_string(); + let mut serializations: Vec = Vec::new(); let mut clones: Vec = Vec::new(); let mut comparisons: Vec = Vec::new(); let mut formats: Vec = Vec::new(); - for variant in &data.variants { + for (variant_index, variant) in (0_u32..).zip(&data.variants) { let ident = &variant.ident; let label = ident.to_string(); + let serialized_label = serde_snake_case(&label); + serializations.push(match &variant.fields { + Fields::Unit => quote! { + Self::#ident => serializer.serialize_unit_variant(#type_label, #variant_index, #serialized_label) + }, + Fields::Unnamed(fields) if fields.unnamed.len() == 1 => quote! { + Self::#ident(own_0) => serializer.serialize_newtype_variant(#type_label, #variant_index, #serialized_label, own_0) + }, + _ => return quote!(compile_error!("StackSafe serializes only unit and one-field variants");), + }); let (pattern, other_pattern, bindings, others, rebuild, format) = match &variant.fields { Fields::Unit => ( quote!(Self::#ident), @@ -100,6 +120,14 @@ pub(crate) fn expand_stack_safe(input: &DeriveInput) -> proc_macro2::TokenStream } } + impl ::serde::Serialize for #name { + fn serialize(&self, serializer: S) -> ::core::result::Result { + crate::expressions::tree::main::_on_grown_stack::on_grown_stack(|| match self { + #(#serializations,)* + }) + } + } + impl ::core::fmt::Debug for #name { fn fmt(&self, formatter: &mut ::core::fmt::Formatter<'_>) -> ::core::fmt::Result { crate::expressions::tree::main::_on_grown_stack::on_grown_stack(|| match self { @@ -109,3 +137,38 @@ pub(crate) fn expand_stack_safe(input: &DeriveInput) -> proc_macro2::TokenStream } } } + +/// Whether the only serde attribute on the enum is `rename_all = "snake_case"`, and none on its variants. +fn has_only_snake_case_serde_attributes(input: &DeriveInput) -> bool { + let Data::Enum(data) = &input.data else { + return false; + }; + let variant_attributes = data.variants.iter().flat_map(|variant| &variant.attrs); + let field_attributes = data + .variants + .iter() + .flat_map(|variant| &variant.fields) + .flat_map(|field| &field.attrs); + if variant_attributes + .chain(field_attributes) + .any(|attribute| attribute.path().is_ident("serde")) + { + return false; + } + let mut snake_case = false; + for attribute in input + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("serde")) + { + let parsed = attribute.parse_nested_meta(|meta| { + let value: syn::LitStr = meta.value()?.parse()?; + snake_case = meta.path.is_ident("rename_all") && value.value() == SNAKE_CASE; + Ok(()) + }); + if parsed.is_err() || !snake_case { + return false; + } + } + snake_case +} diff --git a/crates/sqlquack-syntax-derive/src/ast_node/main/mod.rs b/crates/sqlquack-syntax-derive/src/ast_node/main/mod.rs index 145a9ec2f..0b8b5c74e 100644 --- a/crates/sqlquack-syntax-derive/src/ast_node/main/mod.rs +++ b/crates/sqlquack-syntax-derive/src/ast_node/main/mod.rs @@ -1,3 +1,4 @@ //! Main. pub(crate) mod expand_ast_node; +pub(crate) mod expand_stack_safe; diff --git a/crates/sqlquack-syntax-derive/src/ast_node/mod.rs b/crates/sqlquack-syntax-derive/src/ast_node/mod.rs index 83f5e5fc9..3af398f4f 100644 --- a/crates/sqlquack-syntax-derive/src/ast_node/mod.rs +++ b/crates/sqlquack-syntax-derive/src/ast_node/mod.rs @@ -1,4 +1,4 @@ -//! The `AstNode` derive. +//! The `AstNode` and `StackSafe` derives. pub(crate) mod _helpers; pub(crate) mod main; diff --git a/crates/sqlquack-syntax-derive/src/constants.rs b/crates/sqlquack-syntax-derive/src/constants.rs index 3a51c5f23..0a6093983 100644 --- a/crates/sqlquack-syntax-derive/src/constants.rs +++ b/crates/sqlquack-syntax-derive/src/constants.rs @@ -8,3 +8,6 @@ pub(crate) const SKIP_MARKER: &str = "skip"; /// The field a node keeps its recorded source range in. pub(crate) const SPAN_FIELD: &str = "span"; + +/// The only `rename_all` value `StackSafe` serializes variant names with. +pub(crate) const SNAKE_CASE: &str = "snake_case"; diff --git a/crates/sqlquack-syntax-derive/src/lib.rs b/crates/sqlquack-syntax-derive/src/lib.rs index 8b4ff08d6..daedf53ab 100644 --- a/crates/sqlquack-syntax-derive/src/lib.rs +++ b/crates/sqlquack-syntax-derive/src/lib.rs @@ -1,17 +1,16 @@ -//! Derives exhaustive AST traversal, and stack-safe `Clone`, `PartialEq` and `Debug` for recursive enums. +//! Derives exhaustive AST traversal, and stack-safe `Clone`, `PartialEq`, `Debug` and `Serialize` for recursive enums. #![forbid(unsafe_code)] mod ast_node; mod constants; -mod stack_safe; use proc_macro::TokenStream; use syn::parse_macro_input; use syn::DeriveInput; use crate::ast_node::main::expand_ast_node::expand_ast_node; -use crate::stack_safe::main::expand_stack_safe::expand_stack_safe; +use crate::ast_node::main::expand_stack_safe::expand_stack_safe; #[proc_macro_derive(AstNode, attributes(ast))] /// Derive the expression-child visitors for an AST payload. @@ -21,7 +20,7 @@ pub fn derive_ast_node(input: TokenStream) -> TokenStream { } #[proc_macro_derive(StackSafe)] -/// Derive `Clone`, `PartialEq` and `Debug` for a recursive enum, each on a grown stack when it runs low. +/// Derive `Clone`, `PartialEq`, `Debug` and `Serialize` for a recursive enum, each on a grown stack when it runs low. pub fn derive_stack_safe(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as DeriveInput); expand_stack_safe(&input).into() diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs b/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs deleted file mode 100644 index 60bde10f4..000000000 --- a/crates/sqlquack-syntax-derive/src/stack_safe/main/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Main. - -pub(crate) mod expand_stack_safe; diff --git a/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs b/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs deleted file mode 100644 index ddcf80120..000000000 --- a/crates/sqlquack-syntax-derive/src/stack_safe/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! The `StackSafe` derive. - -pub(crate) mod main; diff --git a/crates/sqlquack-syntax/src/expressions/operators/models.rs b/crates/sqlquack-syntax/src/expressions/operators/models.rs index 2f7e95cc4..b5fdcf95a 100644 --- a/crates/sqlquack-syntax/src/expressions/operators/models.rs +++ b/crates/sqlquack-syntax/src/expressions/operators/models.rs @@ -140,13 +140,7 @@ pub struct Case { /// The payload of every binary operator variant: arithmetic, comparison, logical, bitwise. #[derive(sqlquack_syntax_derive::AstNode, Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BinaryOp { - #[serde( - serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" - )] pub left: Expression, - #[serde( - serialize_with = "crate::expressions::tree::main::_serialize_on_grown_stack::serialize_on_grown_stack" - )] pub right: Expression, /// Comments after the left operand (before the operator) #[serde(default)] diff --git a/crates/sqlquack-syntax/src/expressions/tree/models.rs b/crates/sqlquack-syntax/src/expressions/tree/models.rs index 1d614bf6c..ce40f03b9 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/models.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/models.rs @@ -1,7 +1,6 @@ //! Data models of expressions tree. use serde::Deserialize; -use serde::Serialize; use std::fmt; use crate::expressions::columns::models::AlterColumn; @@ -737,9 +736,7 @@ use crate::expressions::types::models::Struct; use crate::expressions::types::models::Tuple; /// Any SQL expression or statement, as one recursive AST node. -#[derive( - sqlquack_syntax_derive::AstNode, sqlquack_syntax_derive::StackSafe, Serialize, Deserialize, -)] +#[derive(sqlquack_syntax_derive::AstNode, sqlquack_syntax_derive::StackSafe, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Expression { /// Structured Vertica syntax whose semantics are not shared by other dialects. diff --git a/crates/sqlquack-syntax/src/guard/tests/helpers.rs b/crates/sqlquack-syntax/src/guard/tests/helpers.rs index 3e4f6c7c5..903d935b4 100644 --- a/crates/sqlquack-syntax/src/guard/tests/helpers.rs +++ b/crates/sqlquack-syntax/src/guard/tests/helpers.rs @@ -92,9 +92,9 @@ pub(crate) fn constructor_levels(n: usize, open: &str) -> String { ) } -/// Default-guarded Snowflake trees with 100,000-term chains and nesting just inside each depth guard. +/// Snowflake trees with 100,000-term chains, nesting inside each default guard, and 8,000 levels unguarded. pub(crate) fn deep_trees() -> Vec { - let statements = [ + let guarded = [ format!( "SELECT f(1{}) FROM a WHERE a = 1{}", " + 1".repeat(100_000), @@ -111,15 +111,34 @@ pub(crate) fn deep_trees() -> Vec format!("SELECT {}TRUE", "NOT ".repeat(500)), format!("SELECT CAST(NULL AS INT{})", "[]".repeat(512)), ]; + let unguarded = [ + format!("SELECT 1{}", "::INT".repeat(8_000)), + format!( + "SELECT {}1{}", + "CASE WHEN TRUE THEN ".repeat(8_000), + " END".repeat(8_000) + ), + format!("SELECT {}1{}", "(".repeat(8_000), ")".repeat(8_000)), + ]; + let unlimited = Some(usize::MAX); + let raised = crate::ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_parser_depth: unlimited, + max_input_bytes: unlimited, + max_tokens: unlimited, + max_ast_nodes: unlimited, + max_ast_depth: unlimited, + max_parenthesis_depth: unlimited, + max_function_call_depth: unlimited, + }), + }; + let options = std::iter::repeat_n(crate::ParseOptions::default(), guarded.len()) + .chain(std::iter::repeat_n(raised, unguarded.len())); let mut trees = Vec::new(); - for sql in statements { + for (sql, options) in guarded.into_iter().chain(unguarded).zip(options) { trees.extend( - crate::parse_statements( - &sql, - crate::Dialect::Snowflake, - &crate::ParseOptions::default(), - ) - .expect("the statement is inside every guard"), + crate::parse_statements(&sql, crate::Dialect::Snowflake, &options) + .expect("the statement is inside its guards"), ); } trees diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index 8077616ad..c394c98b1 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -272,7 +272,7 @@ fn given_a_deeply_nested_type_when_parsing_on_a_2_mb_thread_then_512_levels_pars } #[test] -fn given_long_chains_and_guard_deep_nesting_on_a_2_mb_thread_when_cloning_comparing_formatting_serializing_or_dropping_then_none_overflows( +fn given_long_chains_and_deep_nesting_on_a_2_mb_thread_when_cloning_comparing_formatting_serializing_or_dropping_then_none_overflows( ) { let test_cases = [ TreeOperationTestCase { @@ -309,9 +309,15 @@ fn given_long_chains_and_guard_deep_nesting_on_a_2_mb_thread_when_cloning_compar ]; for test_case in test_cases { let operation = test_case.operation; + let trees = std::thread::Builder::new() + .stack_size(256 << 20) + .spawn(deep_trees) + .expect("the parsing thread starts") + .join() + .expect("the trees parse"); let completed = std::thread::Builder::new() .stack_size(2 << 20) - .spawn(move || operation(deep_trees())) + .spawn(move || operation(trees)) .expect("the thread starts") .join() .expect("the thread finishes"); From b0f3ce88d6bf543e0574df541a13af57e089271a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:42:48 +0000 Subject: [PATCH 10/22] Postgres parser keeps remembered trees within the depth limit and clones them on a grown stack A remembered parenthesised query or JSON_OBJECT argument list was stored and replayed with a recursive clone, which overflowed the stack on a deep tree, such as a long UNION ALL chain in parentheses, before the statement tree depth was checked. A remembered tree past the depth limit is now refused before it is kept, and every stored or replayed tree is cloned on a stack grown to its depth. --- crates/sqlquack-syntax-postgres/README.md | 4 +- .../src/parse/_helpers/tree/cursor.rs | 35 ++++++++++- .../src/parse/tests/test_depth.rs | 60 +++++++++++++++++++ 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/crates/sqlquack-syntax-postgres/README.md b/crates/sqlquack-syntax-postgres/README.md index 3321ca84a..88c20d99a 100644 --- a/crates/sqlquack-syntax-postgres/README.md +++ b/crates/sqlquack-syntax-postgres/README.md @@ -29,7 +29,9 @@ Nesting is bounded so that no input exhausts the caller's stack or takes exponen (132 levels of 99 casts each) before its own 54001, so a statement 8,000 to 13,200 levels deep is refused here and accepted by the engine. - A parenthesized query, and `JSON_OBJECT`'s arguments, are parsed once from each token and the - outcome replayed, so alternatives that each read the same nested text stay linear. + outcome replayed, so alternatives that each read the same nested text stay linear. A replayed + tree is cloned recursively, so it is cloned on a stack grown to its depth, and a tree past the + 8,000-level limit below is refused before it is kept. ## Gaps diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs index b1c4afc92..f9029423c 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs @@ -22,6 +22,18 @@ const MAX_CHAIN: usize = 1000; /// The deepest statement tree; a release build serialises one this deep on a 2 MiB thread. const MAX_TREE_DEPTH: usize = 8000; +/// Stack a release build's recursive clone of a remembered tree may use per level of its depth. +#[cfg(feature = "stacker")] +const CLONE_LEVEL_BYTES: usize = 2 * 1024; + +/// Stack a debug build's recursive clone of a remembered tree may use per level of its depth. +#[cfg(feature = "stacker")] +const CLONE_LEVEL_BYTES_DEBUG: usize = 8 * 1024; + +/// Stack kept free beyond the clone's own estimate when the stack is grown for it. +#[cfg(feature = "stacker")] +const CLONE_STACK_MARGIN: usize = 1024 * 1024; + /// PostgreSQL's message for 54001, which recursion and chains past their limits both raise. const STACK_DEPTH_EXCEEDED: &str = "stack depth limit exceeded"; @@ -422,12 +434,15 @@ impl<'sql> Parser<'sql> { self.at = outcome.reached; self.opaque |= !outcome.gaps.is_empty(); self.gaps.extend(outcome.gaps.iter().cloned()); - return outcome.result.clone(); + return copied(&outcome.result); } let gaps: usize = self.gaps.len(); - let result: Parsing = parse(self); + let result: Parsing = match parse(self) { + Ok(tree) if tree.tree_depth() > MAX_TREE_DEPTH => Err(self.refuse_depth()), + result => result, + }; let outcome: Outcome = Outcome { - result: result.clone(), + result: copied(&result), reached: self.at, gaps: self.gaps[gaps.min(self.gaps.len())..].to_vec(), }; @@ -519,3 +534,17 @@ impl<'sql> Parser<'sql> { Ok(parts) } } + +/// A copy of a remembered parse; its tree is cloned recursively, so on a stack grown to its depth. +fn copied(result: &Parsing) -> Parsing { + #[cfg(feature = "stacker")] + if let Ok(tree) = result { + let per_level: usize = match cfg!(debug_assertions) { + true => CLONE_LEVEL_BYTES_DEBUG, + false => CLONE_LEVEL_BYTES, + }; + let needed: usize = (tree.tree_depth() + 1) * per_level; + return stacker::maybe_grow(needed, needed + CLONE_STACK_MARGIN, || Ok(tree.clone())); + } + result.clone() +} diff --git a/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs index d6490faec..86331b4d1 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/tests/test_depth.rs @@ -350,6 +350,66 @@ fn given_deep_nesting_when_parsing_on_a_small_stack_then_the_depth_limit_refuses levels: 100000, expected_error: Some(("stack depth limit exceeded", false)), }, + NestingTestCase { + description: "a UNION ALL chain of 2,000 in a parenthesized FROM query", + prefix: "SELECT * FROM (", + open: "SELECT 1 UNION ALL ", + innermost: "SELECT 1) s", + close: "", + read: |text| parse(text).err(), + levels: 2000, + expected_error: None, + }, + NestingTestCase { + description: "a UNION ALL chain of 10,000 in a parenthesized FROM query, past the tree depth", + prefix: "SELECT * FROM (", + open: "SELECT 1 UNION ALL ", + innermost: "SELECT 1) s", + close: "", + read: |text| parse(text).err(), + levels: 10000, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "an AND chain of 7,000 in a parenthesized FROM query", + prefix: "SELECT * FROM (SELECT 1 WHERE true", + open: " AND true", + innermost: ") s", + close: "", + read: |text| parse(text).err(), + levels: 7000, + expected_error: None, + }, + NestingTestCase { + description: "60 parenthesized queries of 60 unions each", + prefix: "", + open: "(", + innermost: "SELECT 1", + close: format!("{})", " UNION ALL SELECT 1".repeat(60)).leak(), + read: |text| parse(text).err(), + levels: 60, + expected_error: None, + }, + NestingTestCase { + description: "60 parenthesized queries of 200 unions each, past the tree depth", + prefix: "", + open: "(", + innermost: "SELECT 1", + close: format!("{})", " UNION ALL SELECT 1".repeat(200)).leak(), + read: |text| parse(text).err(), + levels: 60, + expected_error: Some(("stack depth limit exceeded", false)), + }, + NestingTestCase { + description: "150 parenthesized queries of 60 unions each, past the tree depth", + prefix: "", + open: "(", + innermost: "SELECT 1", + close: format!("{})", " UNION ALL SELECT 1".repeat(60)).leak(), + read: |text| parse(text).err(), + levels: 150, + expected_error: Some(("stack depth limit exceeded", false)), + }, NestingTestCase { description: "an OR chain past the tree depth", prefix: "SELECT true", From 916e56b292120151e418be0388e94c40c18d12f9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:11:15 +0000 Subject: [PATCH 11/22] coverage: probe every native parser for deep-nesting crashes and hangs syntax-coverage depth-probe nests 27 constructs (FROM items, subqueries, parenthesised queries and expressions, CASE, function arguments, casts, subscripts, type suffixes and constructors, and flat chains) at depths from 50 to 100,000, and parses each through the isolated worker on a 2 MiB thread with a 15 s bound. It fails on any crash, panic or timeout coverage/deep-nesting.toml does not list at its depth, and on any listed entry that no longer fails. Each entry names its parser, construct, build profile, depth and owning crate. check-ci runs it on the gate build; make depth-probe-debug runs it on a debug build, whose larger frames fail at shallower depths. --- Makefile | 14 +- coverage/deep-nesting.toml | 434 ++++++++++++++++++ .../src/cli/_helpers/commands.rs | 52 ++- .../src/cli/_helpers/depth_command.rs | 130 ++++++ .../src/cli/_helpers/mod.rs | 1 + .../src/cli/main/run_cli.rs | 12 +- .../src/cli/models.rs | 19 + .../src/depth/_helpers/constructs.rs | 209 +++++++++ .../src/depth/_helpers/mod.rs | 1 + .../src/depth/constants.rs | 18 + .../src/depth/main/compare_allowlist.rs | 80 ++++ .../src/depth/main/mod.rs | 5 + .../src/depth/main/probe_construct.rs | 29 ++ .../src/depth/main/probed_constructs.rs | 14 + .../sqlquack-syntax-coverage/src/depth/mod.rs | 9 + .../src/depth/models.rs | 57 +++ .../src/depth/tests.rs | 8 + .../src/depth/tests/helpers.rs | 66 +++ .../src/depth/tests/test_compare_allowlist.rs | 102 ++++ .../src/depth/tests/test_probe_construct.rs | 104 +++++ .../src/depth/tests/test_types.rs | 31 ++ crates/sqlquack-syntax-coverage/src/lib.rs | 1 + 22 files changed, 1381 insertions(+), 15 deletions(-) create mode 100644 coverage/deep-nesting.toml create mode 100644 crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/_helpers/mod.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/constants.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/main/compare_allowlist.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/main/mod.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/main/probe_construct.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/main/probed_constructs.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/mod.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/models.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/tests.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/tests/helpers.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/tests/test_compare_allowlist.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs create mode 100644 crates/sqlquack-syntax-coverage/src/depth/tests/test_types.rs diff --git a/Makefile b/Makefile index 79c1f5a95..9ebd0fda4 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: format rust-lint python-check fensu syntax-coverage gate-build evidence-guard polyglot-guard generate-cases generated-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-results check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug gate-build evidence-guard polyglot-guard generate-cases generated-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-results check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -62,6 +62,16 @@ fensu: syntax-coverage: gate-build $(GATE_DIR)/syntax-coverage diff coverage/syntax/baseline.json +# Fails when a native parser crashes, panics or hangs on deeply nested SQL on a 2 MiB thread and +# coverage/deep-nesting.toml does not list it, or when a listed failure no longer happens. +depth-probe: gate-build + $(GATE_DIR)/syntax-coverage depth-probe --profile $(GATE_PROFILE) + +# The same probe on a debug build, whose larger frames fail at shallower depths; not in check-ci. +depth-probe-debug: + cargo build --locked -p sqlquack-syntax-coverage --bin syntax-coverage + $(or $(CARGO_TARGET_DIR),target)/debug/syntax-coverage depth-probe --profile dev + # Fails when a change modifies or deletes a recorded result, or adds an invalid one. evidence-guard: uv run python -m scripts.check_evidence_append_only --base $(EVIDENCE_BASE) @@ -97,7 +107,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage evidence-guard polyglot-guard generated-cases referee-cases python-api +check-ci: rust-lint python-check fensu syntax-coverage depth-probe evidence-guard polyglot-guard generated-cases referee-cases python-api # The gate builds everything it runs once, in `gate-build`: one profile, GATE_PROFILE (Cargo.toml's # `gate`), and one feature set, GATE_FEATURES. Cargo unifies features per invocation, so a build of a diff --git a/coverage/deep-nesting.toml b/coverage/deep-nesting.toml new file mode 100644 index 000000000..7350039b2 --- /dev/null +++ b/coverage/deep-nesting.toml @@ -0,0 +1,434 @@ +# Deep-nesting failures the depth probe tolerates until their owners fix them. +# +# `syntax-coverage depth-probe` parses every construct at each ladder depth on a 2 MiB thread, in +# every native parser, and fails on a crash, panic or 15 s timeout no entry here names, and on an +# entry whose construct no longer fails. A parser is fit for in-process use (native routes, lint, +# the formatter) only once it has no entry. Each fix removes its own entries in the same change; +# each new parser adds its entries, or its depth fix, in the change that links it. +# +# `depth` is the shallowest ladder depth that fails; `profile` is the Cargo profile probed: `gate` +# in check-ci, `dev` in `make depth-probe-debug`. + +[[known_failures]] +parser = "snowflake" +construct = "type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "type_suffix_sized" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "column_type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "flat_plus" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "postgres" +construct = "from_parentheses" +profile = "gate" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "from_parenthesised_join" +profile = "gate" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "from_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "scalar_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "exists_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "in_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_query" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_set_operation" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "nested_with" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_expression" +profile = "gate" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "case" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "function_arguments" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "cast" +profile = "gate" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "subscript" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "type_suffix_sized" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "double_colon_type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "column_type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "flat_plus" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "flat_or" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "postgres" +construct = "flat_union_all" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "snowflake" +construct = "type_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "type_suffix_sized" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "column_type_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "type_constructor_and_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax: Snowflake parser" + +[[known_failures]] +parser = "snowflake" +construct = "flat_plus" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "postgres" +construct = "from_parentheses" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "from_parenthesised_join" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "from_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "scalar_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "exists_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "in_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_query" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_set_operation" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "nested_with" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "parenthesised_expression" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "unary_minus" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "not" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "case" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "function_arguments" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "cast" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "subscript" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "type_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "type_suffix_sized" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "double_colon_type_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "column_type_suffix" +profile = "dev" +depth = 1000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "flat_plus" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "flat_or" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "postgres" +construct = "flat_union_all" +profile = "dev" +depth = 3000 +failure = "crash" +owner = "sqlquack-syntax: shared tree serialisation and drop" diff --git a/crates/sqlquack-syntax-coverage/src/cli/_helpers/commands.rs b/crates/sqlquack-syntax-coverage/src/cli/_helpers/commands.rs index 7c7eeca3c..3cd0bf44a 100644 --- a/crates/sqlquack-syntax-coverage/src/cli/_helpers/commands.rs +++ b/crates/sqlquack-syntax-coverage/src/cli/_helpers/commands.rs @@ -47,17 +47,23 @@ const BOUNDARY_KINDS: [ObligationKind; 4] = [ /// The time one local parse may take before it fails as a timeout. const PARSE_BOUND: Duration = Duration::from_secs(10); +/// The exit code of a Rust process whose main thread panicked, which the worker reads as a panic. +const PANIC_EXIT: u8 = 101; /// The worker's virtual memory bound in KiB, 4 GiB; exceeding it fails the case as a crash. const PARSE_MEMORY_KIB: u64 = 4 * 1024 * 1024; -/// The worker command: this binary, its memory bounded by the shell before it starts. -fn worker_command(binary: PathBuf, local_parser: LocalParser) -> Box Command> { - let worker: String = format!( - "parse-worker --parser {}", - local_parser - .to_possible_value() - .map_or_else(String::new, |value| value.get_name().to_owned()) - ); +/// The worker command: this binary, memory-bounded by the shell, parsing on a `thread_stack` thread. +pub(crate) fn worker_command( + binary: PathBuf, + local_parser: LocalParser, + thread_stack: Option, +) -> Box Command> { + let parser: String = local_parser + .to_possible_value() + .map_or_else(String::new, |value| value.get_name().to_owned()); + let stack: String = + thread_stack.map_or_else(String::new, |bytes| format!(" --thread-stack {bytes}")); + let worker: String = format!("parse-worker --parser {parser}{stack}"); Box::new(move || { let mut command = Command::new("sh"); command @@ -99,7 +105,7 @@ fn checked(coverage: &SyntaxCoverage) -> Result { })?; let local_parser: LocalParser = current_dialect().local_parser; let worker: RefCell = RefCell::new(Worker::new( - worker_command(binary, local_parser), + worker_command(binary, local_parser, None), PARSE_BOUND, )); let parses: bool = local_verdict(local_parser).is_some(); @@ -522,15 +528,37 @@ pub(crate) fn diff( }) } +/// `verdict_of(sql)` on a thread with `bytes` of stack; a panic there exits as a panicking worker. +fn on_thread( + bytes: usize, + verdict_of: fn(&str) -> LocalVerdict, + sql: &str, +) -> Result { + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(bytes) + .spawn_scoped(scope, || verdict_of(sql)) + .map_err(|_| ExitCode::FAILURE)? + .join() + .map_err(|_| ExitCode::from(PANIC_EXIT)) + }) +} + /// Reads one JSON string of SQL per line and prints the local verdict on each, flushed. -pub(crate) fn parse_worker(parser: LocalParser) -> Result { +pub(crate) fn parse_worker( + parser: LocalParser, + thread_stack: Option, +) -> Result { let verdict_of: fn(&str) -> LocalVerdict = local_verdict(parser).ok_or(ExitCode::FAILURE)?; let stdout = std::io::stdout(); for line in std::io::stdin().lock().lines() { let line: String = line.map_err(|_| ExitCode::FAILURE)?; let sql: String = serde_json::from_str(&line).map_err(|_| ExitCode::FAILURE)?; - let verdict: String = - serde_json::to_string(&verdict_of(&sql)).map_err(|_| ExitCode::FAILURE)?; + let verdict: LocalVerdict = match thread_stack { + None => verdict_of(&sql), + Some(bytes) => on_thread(bytes, verdict_of, &sql)?, + }; + let verdict: String = serde_json::to_string(&verdict).map_err(|_| ExitCode::FAILURE)?; let mut out = stdout.lock(); writeln!(out, "{verdict}") .and_then(|()| out.flush()) diff --git a/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs b/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs new file mode 100644 index 000000000..d67ef7aad --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs @@ -0,0 +1,130 @@ +//! `depth-probe`: every native parser's deep-nesting failures against the allowlist. + +use std::path::{Path, PathBuf}; +use std::process::ExitCode; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +use serde_json::json; + +use crate::cli::_helpers::commands::worker_command; +use crate::cli::_helpers::output::{print, say}; +use crate::cli::_helpers::worker::Worker; +use crate::depth::constants::{PROBE_BOUND, PROBE_STACK_BYTES, PROBED_PARSERS}; +use crate::depth::main::compare_allowlist::compare_allowlist; +use crate::depth::main::probe_construct::probe_construct; +use crate::depth::main::probed_constructs::probed_constructs; +use crate::depth::models::{Allowlist, Construct, ConstructResult, DepthReport}; +use crate::formats::models::LocalParser; + +fn read_allowlist(path: &Path) -> Result { + std::fs::read_to_string(path) + .map_err(|error| error.to_string()) + .and_then(|text| toml::from_str(&text).map_err(|error| error.to_string())) + .map_err(|problem| { + say(&format!("FAILED, {}: {problem}", path.display())); + ExitCode::FAILURE + }) +} + +/// Every construct of every probed parser, probed by `jobs` threads each with its own worker. +fn probe_all(binary: &Path, jobs: usize) -> Vec { + let mut tasks: Vec<(LocalParser, Construct)> = Vec::new(); + for parser in PROBED_PARSERS { + for construct in probed_constructs(parser) { + tasks.push((parser, construct)); + } + } + let next: AtomicUsize = AtomicUsize::new(0); + let results: Mutex>> = Mutex::new(vec![None; tasks.len()]); + std::thread::scope(|scope| { + for _ in 0..jobs.max(1) { + scope.spawn(|| { + loop { + let index: usize = next.fetch_add(1, Ordering::SeqCst); + let Some(&(parser, construct)) = tasks.get(index) else { + break; + }; + let mut worker: Worker = Worker::new( + worker_command(binary.to_path_buf(), parser, Some(PROBE_STACK_BYTES)), + PROBE_BOUND, + ); + let result: ConstructResult = + probe_construct(parser, &construct, &mut |sql| worker.verdict(sql)); + if let Ok(mut all) = results.lock() + && let Some(slot) = all.get_mut(index) + { + *slot = Some(result); + } + } + }); + } + }); + results + .into_inner() + .unwrap_or_default() + .into_iter() + .flatten() + .collect() +} + +fn result_line(result: &ConstructResult) -> String { + let outcome: String = match result.failed_at { + Some((depth, failure)) => format!("{failure:?} at depth {depth}").to_lowercase(), + None => "clean at every depth".to_owned(), + }; + format!( + "{} {}: {outcome}", + format!("{:?}", result.parser).to_lowercase(), + result.construct + ) +} + +pub(crate) fn depth_probe( + profile: &str, + allowlist: &Path, + jobs: usize, + json: bool, +) -> Result { + let known: Allowlist = read_allowlist(allowlist)?; + let binary: PathBuf = std::env::current_exe().map_err(|problem| { + say(&format!( + "FAILED, cannot find this binary for the parse worker: {problem}" + )); + ExitCode::FAILURE + })?; + let started: Instant = Instant::now(); + let results: Vec = probe_all(&binary, jobs); + let report: DepthReport = compare_allowlist(&results, &known, profile); + let failing: usize = results + .iter() + .filter(|result| result.failed_at.is_some()) + .count(); + let mut text: Vec = results.iter().map(result_line).collect(); + text.extend( + report + .unlisted + .iter() + .map(|line| format!("UNLISTED {line}")), + ); + text.extend(report.stale.iter().map(|line| format!("STALE {line}"))); + text.extend(report.problems.iter().map(|line| format!("PROBLEM {line}"))); + print( + json, + &json!({ "profile": profile, "results": results, "report": report }), + &text, + ); + say(&format!( + "probed {} constructs in profile {profile} in {:.1} s: {failing} fail, {} unlisted, {} stale, {} problems", + results.len(), + started.elapsed().as_secs_f64(), + report.unlisted.len(), + report.stale.len(), + report.problems.len() + )); + match report == DepthReport::default() { + true => Ok(ExitCode::SUCCESS), + false => Err(ExitCode::FAILURE), + } +} diff --git a/crates/sqlquack-syntax-coverage/src/cli/_helpers/mod.rs b/crates/sqlquack-syntax-coverage/src/cli/_helpers/mod.rs index 6a5f87d73..fd3442ff6 100644 --- a/crates/sqlquack-syntax-coverage/src/cli/_helpers/mod.rs +++ b/crates/sqlquack-syntax-coverage/src/cli/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod commands; +pub(crate) mod depth_command; pub(crate) mod local; pub(crate) mod output; pub(crate) mod results_command; diff --git a/crates/sqlquack-syntax-coverage/src/cli/main/run_cli.rs b/crates/sqlquack-syntax-coverage/src/cli/main/run_cli.rs index 96365e22b..f03ed94f4 100644 --- a/crates/sqlquack-syntax-coverage/src/cli/main/run_cli.rs +++ b/crates/sqlquack-syntax-coverage/src/cli/main/run_cli.rs @@ -8,6 +8,7 @@ use crate::cli::_helpers::commands::{ boundaries, diff, family, missing, pages, parse_worker, snapshot_command, stale, summary, translation, validate, }; +use crate::cli::_helpers::depth_command::depth_probe; use crate::cli::_helpers::results_command::results; use crate::cli::models::{Arguments, Command}; @@ -41,7 +42,16 @@ pub fn run_cli() -> ExitCode { &common, ), Command::Diff { base, head, common } => diff(&base, head.as_deref(), &common), - Command::ParseWorker { parser } => parse_worker(parser), + Command::ParseWorker { + parser, + thread_stack, + } => parse_worker(parser, thread_stack), + Command::DepthProbe { + profile, + allowlist, + jobs, + json, + } => depth_probe(&profile, &allowlist, jobs, json), }; result.unwrap_or_else(|code| code) } diff --git a/crates/sqlquack-syntax-coverage/src/cli/models.rs b/crates/sqlquack-syntax-coverage/src/cli/models.rs index 9ca51b18c..fb35f3841 100644 --- a/crates/sqlquack-syntax-coverage/src/cli/models.rs +++ b/crates/sqlquack-syntax-coverage/src/cli/models.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use clap::{Args, Parser, Subcommand}; +use crate::depth::constants::ALLOWLIST_FILE; use crate::formats::constants::COVERAGE_ROOT; use crate::formats::models::LocalParser; @@ -107,6 +108,24 @@ pub enum Command { /// The dialect parser that gives the verdicts. #[arg(long, value_enum)] parser: LocalParser, + /// Parse each text on a thread with this many bytes of stack. + #[arg(long)] + thread_stack: Option, + }, + /// Deep nesting in every native parser; fails on an unlisted crash or hang, or a stale entry. + DepthProbe { + /// The Cargo profile this binary was built with; it selects the allowlist entries. + #[arg(long)] + profile: String, + /// The known failures. + #[arg(long, default_value = ALLOWLIST_FILE)] + allowlist: PathBuf, + /// Parse workers run at once. + #[arg(long, default_value_t = 4)] + jobs: usize, + /// Print JSON on stdout alone. + #[arg(long)] + json: bool, }, /// What conversion makes of every case Snowflake parsed, by form and family. Translation { diff --git a/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs new file mode 100644 index 000000000..aa1db3f81 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs @@ -0,0 +1,209 @@ +//! Every construct the probe nests: recursive paths through queries, expressions and types, and flat chains. + +use crate::depth::constants::MIXED_SUFFIX_RUN; +use crate::depth::models::Construct; +use crate::formats::models::LocalParser; + +/// `open` `depth` times, then `inner`, then `close` `depth` times. +fn nest(open: &str, inner: &str, close: &str, depth: usize) -> String { + format!("{}{inner}{}", open.repeat(depth), close.repeat(depth)) +} + +/// `depth` copies of `term` joined by `separator`. +fn chain(term: &str, separator: &str, depth: usize) -> String { + vec![term; depth].join(separator) +} + +/// A type built by the parser's own nested type constructor, where it has one. +fn constructed_type(parser: LocalParser, depth: usize) -> Option { + match parser { + LocalParser::Snowflake => Some(nest("ARRAY(", "INT", ")", depth)), + LocalParser::Postgres | LocalParser::BigQuery | LocalParser::None => None, + } +} + +/// Constructor levels alternating with runs of `[]`, about `depth` type levels in all. +fn mixed_type(parser: LocalParser, depth: usize) -> Option { + let levels: usize = (depth / MIXED_SUFFIX_RUN).max(1); + let run: String = "[]".repeat(MIXED_SUFFIX_RUN); + match parser { + LocalParser::Snowflake => Some(format!( + "{}INT{}{run}", + "ARRAY(".repeat(levels), + format!("{run})").repeat(levels) + )), + LocalParser::Postgres | LocalParser::BigQuery | LocalParser::None => None, + } +} + +/// The probed constructs, in report order. +pub(crate) fn constructs() -> Vec { + vec![ + Construct { + name: "from_parentheses", + text: |_, depth| Some(format!("SELECT * FROM {}", nest("(", "t", ")", depth))), + }, + Construct { + name: "from_parenthesised_join", + text: |_, depth| { + Some(format!( + "SELECT * FROM {}t{}", + "(".repeat(depth), + " JOIN t ON 1 = 1)".repeat(depth) + )) + }, + }, + Construct { + name: "from_subquery", + text: |_, depth| { + Some(format!( + "SELECT * FROM {}", + nest("(SELECT * FROM ", "t", ")", depth) + )) + }, + }, + Construct { + name: "scalar_subquery", + text: |_, depth| Some(format!("SELECT {}", nest("(SELECT ", "1", ")", depth))), + }, + Construct { + name: "exists_subquery", + text: |_, depth| { + Some(format!( + "SELECT 1 FROM t WHERE {}", + nest("EXISTS (SELECT 1 FROM t WHERE ", "1 = 1", ")", depth) + )) + }, + }, + Construct { + name: "in_subquery", + text: |_, depth| { + Some(format!( + "SELECT 1 FROM t WHERE a IN {}", + nest("(SELECT a FROM t WHERE a IN ", "(1)", ")", depth) + )) + }, + }, + Construct { + name: "parenthesised_query", + text: |_, depth| Some(nest("(", "SELECT 1", ")", depth)), + }, + Construct { + name: "parenthesised_set_operation", + text: |_, depth| { + Some(format!( + "{}SELECT 1{}", + "(".repeat(depth), + ") UNION ALL (SELECT 1)".repeat(depth) + )) + }, + }, + Construct { + name: "nested_with", + text: |_, depth| { + Some(format!( + "WITH {}", + nest( + "a AS (WITH ", + "b AS (SELECT 1) SELECT 1", + ") SELECT 1", + depth + ) + )) + }, + }, + Construct { + name: "parenthesised_expression", + text: |_, depth| Some(format!("SELECT {}", nest("(", "1", ")", depth))), + }, + Construct { + name: "unary_minus", + text: |_, depth| Some(format!("SELECT {}1", "- ".repeat(depth))), + }, + Construct { + name: "not", + text: |_, depth| Some(format!("SELECT {}TRUE", "NOT ".repeat(depth))), + }, + Construct { + name: "case", + text: |_, depth| { + Some(format!( + "SELECT {} FROM t", + nest("CASE WHEN a = 1 THEN ", "1", " END", depth) + )) + }, + }, + Construct { + name: "function_arguments", + text: |_, depth| Some(format!("SELECT {}", nest("coalesce(", "1", ")", depth))), + }, + Construct { + name: "cast", + text: |_, depth| Some(format!("SELECT {}", nest("CAST(", "1", " AS INT)", depth))), + }, + Construct { + name: "subscript", + text: |_, depth| Some(format!("SELECT {} FROM t", nest("a[", "0", "]", depth))), + }, + Construct { + name: "type_suffix", + text: |_, depth| Some(format!("SELECT CAST(NULL AS INT{})", "[]".repeat(depth))), + }, + Construct { + name: "type_suffix_sized", + text: |_, depth| Some(format!("SELECT CAST(NULL AS INT{})", "[3]".repeat(depth))), + }, + Construct { + name: "double_colon_type_suffix", + text: |_, depth| Some(format!("SELECT NULL::INT{}", "[]".repeat(depth))), + }, + Construct { + name: "column_type_suffix", + text: |_, depth| Some(format!("CREATE TABLE t (c INT{})", "[]".repeat(depth))), + }, + Construct { + name: "type_constructor", + text: |parser, depth| { + constructed_type(parser, depth).map(|kind| format!("SELECT CAST(NULL AS {kind})")) + }, + }, + Construct { + name: "column_type_constructor", + text: |parser, depth| { + constructed_type(parser, depth).map(|kind| format!("CREATE TABLE t (c {kind})")) + }, + }, + Construct { + name: "type_constructor_and_suffix", + text: |parser, depth| { + mixed_type(parser, depth).map(|kind| format!("SELECT CAST(NULL AS {kind})")) + }, + }, + Construct { + name: "flat_plus", + text: |_, depth| Some(format!("SELECT {}", chain("1", " + ", depth))), + }, + Construct { + name: "flat_or", + text: |_, depth| { + Some(format!( + "SELECT 1 FROM t WHERE {}", + chain("a = 1", " OR ", depth) + )) + }, + }, + Construct { + name: "flat_union_all", + text: |_, depth| Some(chain("SELECT 1", " UNION ALL ", depth)), + }, + Construct { + name: "flat_join", + text: |_, depth| { + Some(format!( + "SELECT * FROM t{}", + " JOIN t ON 1 = 1".repeat(depth) + )) + }, + }, + ] +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/_helpers/mod.rs b/crates/sqlquack-syntax-coverage/src/depth/_helpers/mod.rs new file mode 100644 index 000000000..3b26e35d4 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constructs; diff --git a/crates/sqlquack-syntax-coverage/src/depth/constants.rs b/crates/sqlquack-syntax-coverage/src/depth/constants.rs new file mode 100644 index 000000000..37562f7d3 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/constants.rs @@ -0,0 +1,18 @@ +//! The deep-nesting probe's ladder, bounds and allowlist path. + +use std::time::Duration; + +use crate::formats::models::LocalParser; + +/// The known failures of today's parsers, relative to the repository. +pub const ALLOWLIST_FILE: &str = "coverage/deep-nesting.toml"; +/// The nesting depths each construct is parsed at, shallowest first; a construct stops at its first failure. +pub const DEPTH_LADDER: [usize; 6] = [50, 200, 1_000, 3_000, 10_000, 100_000]; +/// The time one probe parse may take before it fails as a timeout. +pub const PROBE_BOUND: Duration = Duration::from_secs(15); +/// The stack of the thread each probe parse runs on, the smallest an in-process caller is assumed to give. +pub const PROBE_STACK_BYTES: usize = 2 * 1024 * 1024; +/// How many runs of `[]` a mixed type nests between its constructor levels. +pub const MIXED_SUFFIX_RUN: usize = 60; +/// Every native parser this checker links, each probed in turn. +pub const PROBED_PARSERS: [LocalParser; 2] = [LocalParser::Snowflake, LocalParser::Postgres]; diff --git a/crates/sqlquack-syntax-coverage/src/depth/main/compare_allowlist.rs b/crates/sqlquack-syntax-coverage/src/depth/main/compare_allowlist.rs new file mode 100644 index 000000000..e8af0c56b --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/main/compare_allowlist.rs @@ -0,0 +1,80 @@ +//! `compare_allowlist`: a probe run against the known failures of one build profile. + +use std::collections::BTreeSet; +use std::fmt::Debug; + +use crate::depth::models::{Allowlist, ConstructResult, DepthReport, KnownFailure}; + +/// A one-word parser or failure as the allowlist writes it. +fn name(value: T) -> String { + format!("{value:?}").to_lowercase() +} + +fn listing(entry: &KnownFailure) -> String { + format!( + "{} at depth {} (owner: {})", + name(entry.failure), + entry.depth, + entry.owner + ) +} + +/// Unlisted failures, stale entries for `profile`, and malformed entries of any profile. +pub fn compare_allowlist( + results: &[ConstructResult], + allowlist: &Allowlist, + profile: &str, +) -> DepthReport { + let mut report: DepthReport = DepthReport::default(); + let probed: BTreeSet<(String, String)> = results + .iter() + .map(|result| (name(result.parser), result.construct.clone())) + .collect(); + let mut seen: BTreeSet<(String, String, String)> = BTreeSet::new(); + for entry in &allowlist.known_failures { + let (parser, construct, entry_profile) = ( + name(entry.parser), + entry.construct.clone(), + entry.profile.clone(), + ); + if !probed.contains(&(parser.clone(), construct.clone())) { + report.problems.push(format!( + "{parser} {construct}: listed, but the probe runs no such construct in {parser}" + )); + } + if !seen.insert((parser.clone(), construct.clone(), entry_profile.clone())) { + report.problems.push(format!( + "{parser} {construct}: listed twice for profile {entry_profile}" + )); + } + } + for result in results { + let (parser, construct) = (name(result.parser), &result.construct); + let entry: Option<&KnownFailure> = allowlist.known_failures.iter().find(|entry| { + entry.parser == result.parser + && entry.construct == result.construct + && entry.profile == profile + }); + match (result.failed_at, entry) { + (Some((depth, failure)), None) => report.unlisted.push(format!( + "{parser} {construct}: {} at depth {depth}, and no entry lists it", + name(failure) + )), + (Some((depth, failure)), Some(entry)) + if (depth, failure) != (entry.depth, entry.failure) => + { + report.unlisted.push(format!( + "{parser} {construct}: {} at depth {depth}, but listed as {}; update the entry", + name(failure), + listing(entry) + )); + } + (None, Some(entry)) => report.stale.push(format!( + "{parser} {construct}: listed as {}, but it now ends cleanly at every depth; remove the entry", + listing(entry) + )), + _ => {} + } + } + report +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/main/mod.rs b/crates/sqlquack-syntax-coverage/src/depth/main/mod.rs new file mode 100644 index 000000000..34a87833f --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/main/mod.rs @@ -0,0 +1,5 @@ +//! Entry points. + +pub mod compare_allowlist; +pub mod probe_construct; +pub mod probed_constructs; diff --git a/crates/sqlquack-syntax-coverage/src/depth/main/probe_construct.rs b/crates/sqlquack-syntax-coverage/src/depth/main/probe_construct.rs new file mode 100644 index 000000000..6f4bf2fed --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/main/probe_construct.rs @@ -0,0 +1,29 @@ +//! `probe_construct`: one construct parsed at each ladder depth until it first fails. + +use crate::check::models::{LocalFailure, LocalVerdict}; +use crate::depth::constants::DEPTH_LADDER; +use crate::depth::models::{Construct, ConstructResult}; +use crate::formats::models::LocalParser; + +/// Where `construct` first crashes, panics or times out in `verdict`; a refusal ends cleanly. +pub fn probe_construct( + parser: LocalParser, + construct: &Construct, + verdict: &mut dyn FnMut(&str) -> LocalVerdict, +) -> ConstructResult { + let failed_at: Option<(usize, LocalFailure)> = DEPTH_LADDER.iter().find_map(|&depth| { + let text: String = (construct.text)(parser, depth)?; + match verdict(&text) { + LocalVerdict::Failed { + failure: + failure @ (LocalFailure::Crash | LocalFailure::Panic | LocalFailure::Timeout), + } => Some((depth, failure)), + _ => None, + } + }); + ConstructResult { + parser, + construct: construct.name.to_owned(), + failed_at, + } +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/main/probed_constructs.rs b/crates/sqlquack-syntax-coverage/src/depth/main/probed_constructs.rs new file mode 100644 index 000000000..4ff553ca1 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/main/probed_constructs.rs @@ -0,0 +1,14 @@ +//! `probed_constructs`: every construct a parser has the syntax for. + +use crate::depth::_helpers::constructs::constructs; +use crate::depth::constants::DEPTH_LADDER; +use crate::depth::models::Construct; +use crate::formats::models::LocalParser; + +/// The constructs `parser` is probed with, in report order. +pub fn probed_constructs(parser: LocalParser) -> Vec { + constructs() + .into_iter() + .filter(|construct| (construct.text)(parser, DEPTH_LADDER[0]).is_some()) + .collect() +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/mod.rs b/crates/sqlquack-syntax-coverage/src/depth/mod.rs new file mode 100644 index 000000000..529378a2f --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/mod.rs @@ -0,0 +1,9 @@ +//! The deep-nesting probe: native parsers must refuse deep input, never crash or hang on it. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-syntax-coverage/src/depth/models.rs b/crates/sqlquack-syntax-coverage/src/depth/models.rs new file mode 100644 index 000000000..97fce2604 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/models.rs @@ -0,0 +1,57 @@ +//! Nested constructs, the allowlist of known failures, and what one probe run found. + +use serde::{Deserialize, Serialize}; + +use crate::check::models::LocalFailure; +use crate::formats::models::LocalParser; + +/// One construct nested to a depth: its name, and its text in a parser that has that syntax. +#[derive(Clone, Copy, Debug)] +pub struct Construct { + pub name: &'static str, + /// The SQL text at a depth, or none when the parser has no such syntax. + pub text: fn(LocalParser, usize) -> Option, +} + +/// `coverage/deep-nesting.toml`: every failure the probe tolerates until its owner fixes it. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Allowlist { + #[serde(default)] + pub known_failures: Vec, +} + +/// A construct that crashes or hangs in one parser and build profile today. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct KnownFailure { + pub parser: LocalParser, + pub construct: String, + /// The Cargo profile the probe was built with: `gate` or `dev`. + pub profile: String, + /// The shallowest ladder depth at which the construct fails. + pub depth: usize, + pub failure: LocalFailure, + /// The crate, and the part of it, whose fix removes this entry. + pub owner: String, +} + +/// Where one construct first failed in one parser, if anywhere. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ConstructResult { + pub parser: LocalParser, + pub construct: String, + /// The shallowest failing ladder depth and how it failed; none when every depth ends cleanly. + pub failed_at: Option<(usize, LocalFailure)>, +} + +/// A probe run held against the allowlist; the gate passes only when every list is empty. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct DepthReport { + /// Failures no entry names, or that an entry names at another depth or failure. + pub unlisted: Vec, + /// Entries for this profile whose construct now ends cleanly at every depth. + pub stale: Vec, + /// Entries naming a construct the probe does not run, or naming one twice. + pub problems: Vec, +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests.rs b/crates/sqlquack-syntax-coverage/src/depth/tests.rs new file mode 100644 index 000000000..854986ab0 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +pub(crate) mod helpers; +#[path = "tests/test_compare_allowlist.rs"] +mod test_compare_allowlist; +#[path = "tests/test_probe_construct.rs"] +mod test_probe_construct; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests/helpers.rs b/crates/sqlquack-syntax-coverage/src/depth/tests/helpers.rs new file mode 100644 index 000000000..e79c1c857 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/tests/helpers.rs @@ -0,0 +1,66 @@ +use crate::check::models::{LocalFailure, LocalVerdict}; +use crate::depth::models::{Construct, ConstructResult, KnownFailure}; +use crate::formats::models::LocalParser; + +/// One parenthesis per level, so a verdict can read the depth from the text's length. +pub(crate) const PARENTHESES: Construct = Construct { + name: "parentheses", + text: |_, depth| Some(format!("{}1{}", "(".repeat(depth), ")".repeat(depth))), +}; + +pub(crate) fn failed(failure: LocalFailure) -> LocalVerdict { + LocalVerdict::Failed { failure } +} + +pub(crate) fn accepted() -> LocalVerdict { + LocalVerdict::Accepted { + opaque: false, + tree_sha256: None, + } +} + +/// `deep` for texts longer than `threshold` bytes, `shallow` for the rest. +pub(crate) fn by_length( + threshold: usize, + deep: &LocalVerdict, + shallow: &LocalVerdict, + sql: &str, +) -> LocalVerdict { + [shallow, deep][usize::from(sql.len() > threshold)].clone() +} + +pub(crate) fn result( + parser: LocalParser, + construct: &str, + failed_at: Option<(usize, LocalFailure)>, +) -> ConstructResult { + ConstructResult { + parser, + construct: construct.to_owned(), + failed_at, + } +} + +pub(crate) fn entry( + parser: LocalParser, + construct: &str, + profile: &str, + depth: usize, +) -> KnownFailure { + KnownFailure { + parser, + construct: construct.to_owned(), + profile: profile.to_owned(), + depth, + failure: LocalFailure::Crash, + owner: "sqlquack-syntax-postgres".to_owned(), + } +} + +/// The `parser construct` each report line is about. +pub(crate) fn subjects(lines: &[String]) -> Vec<&str> { + lines + .iter() + .map(|line| line.split(':').next().unwrap_or_default()) + .collect() +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests/test_compare_allowlist.rs b/crates/sqlquack-syntax-coverage/src/depth/tests/test_compare_allowlist.rs new file mode 100644 index 000000000..462e04856 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/tests/test_compare_allowlist.rs @@ -0,0 +1,102 @@ +use crate::check::models::LocalFailure; +use crate::depth::main::compare_allowlist::compare_allowlist; +use crate::depth::models::Allowlist; +use crate::depth::tests::helpers::{entry, result, subjects}; +use crate::depth::tests::test_types::AllowlistTestCase; +use crate::formats::models::LocalParser; + +#[test] +fn given_an_allowlist_when_comparing_a_probe_run_then_unlisted_and_stale_entries_fail() { + use LocalParser::{Postgres, Snowflake}; + let results = [ + result(Snowflake, "flat_plus", Some((10_000, LocalFailure::Crash))), + result(Postgres, "case", Some((50, LocalFailure::Crash))), + result(Postgres, "not", None), + ]; + let test_cases = [ + AllowlistTestCase { + description: "every failure listed at its depth", + entries: vec![ + entry(Snowflake, "flat_plus", "gate", 10_000), + entry(Postgres, "case", "gate", 50), + ], + expected_unlisted: vec![], + expected_stale: vec![], + expected_problems: vec![], + }, + AllowlistTestCase { + description: "a failure no entry lists", + entries: vec![entry(Snowflake, "flat_plus", "gate", 10_000)], + expected_unlisted: vec!["postgres case"], + expected_stale: vec![], + expected_problems: vec![], + }, + AllowlistTestCase { + description: "a failure listed at a shallower depth than it now fails at", + entries: vec![ + entry(Snowflake, "flat_plus", "gate", 3_000), + entry(Postgres, "case", "gate", 50), + ], + expected_unlisted: vec!["snowflake flat_plus"], + expected_stale: vec![], + expected_problems: vec![], + }, + AllowlistTestCase { + description: "a listed construct that now ends cleanly is stale", + entries: vec![ + entry(Snowflake, "flat_plus", "gate", 10_000), + entry(Postgres, "case", "gate", 50), + entry(Postgres, "not", "gate", 1_000), + ], + expected_unlisted: vec![], + expected_stale: vec!["postgres not"], + expected_problems: vec![], + }, + AllowlistTestCase { + description: "another profile's entries neither excuse nor go stale", + entries: vec![ + entry(Snowflake, "flat_plus", "dev", 10_000), + entry(Postgres, "case", "dev", 50), + entry(Postgres, "not", "dev", 1_000), + ], + expected_unlisted: vec!["snowflake flat_plus", "postgres case"], + expected_stale: vec![], + expected_problems: vec![], + }, + AllowlistTestCase { + description: "an entry the probe cannot check, and one listed twice", + entries: vec![ + entry(Snowflake, "flat_plus", "gate", 10_000), + entry(Postgres, "case", "gate", 50), + entry(Postgres, "case", "gate", 50), + entry(Postgres, "type_constructor", "dev", 50), + ], + expected_unlisted: vec![], + expected_stale: vec![], + expected_problems: vec!["postgres case", "postgres type_constructor"], + }, + ]; + for test_case in test_cases { + let report = compare_allowlist( + &results, + &Allowlist { + known_failures: test_case.entries, + }, + "gate", + ); + assert_eq!( + ( + subjects(&report.unlisted), + subjects(&report.stale), + subjects(&report.problems) + ), + ( + test_case.expected_unlisted, + test_case.expected_stale, + test_case.expected_problems + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs b/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs new file mode 100644 index 000000000..3768f2b50 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs @@ -0,0 +1,104 @@ +use crate::check::models::{LocalFailure, LocalVerdict}; +use crate::depth::main::probe_construct::probe_construct; +use crate::depth::main::probed_constructs::probed_constructs; +use crate::depth::tests::helpers::{PARENTHESES, accepted, by_length, failed}; +use crate::depth::tests::test_types::{ConstructsTestCase, ProbeTestCase}; +use crate::formats::models::LocalParser; + +#[test] +fn given_a_parser_that_fails_past_some_depth_when_probing_then_the_first_failing_depth_is_named() { + let test_cases = [ + ProbeTestCase { + description: "a crash from depth 1,000 stops the ladder there", + threshold: 2_000, + deep: failed(LocalFailure::Crash), + shallow: LocalVerdict::Rejected, + expected_failed_at: Some((1_000, LocalFailure::Crash)), + expected_parses: 3, + }, + ProbeTestCase { + description: "a hang at the shallowest depth", + threshold: 0, + deep: failed(LocalFailure::Timeout), + shallow: LocalVerdict::Rejected, + expected_failed_at: Some((50, LocalFailure::Timeout)), + expected_parses: 1, + }, + ProbeTestCase { + description: "a panic counts as a failure", + threshold: 10_000, + deep: failed(LocalFailure::Panic), + shallow: accepted(), + expected_failed_at: Some((10_000, LocalFailure::Panic)), + expected_parses: 5, + }, + ProbeTestCase { + description: "a complexity guard's refusal ends cleanly at every depth", + threshold: 0, + deep: failed(LocalFailure::Guard), + shallow: accepted(), + expected_failed_at: None, + expected_parses: 6, + }, + ProbeTestCase { + description: "acceptance below a depth refusal ends cleanly", + threshold: 500, + deep: LocalVerdict::Rejected, + shallow: accepted(), + expected_failed_at: None, + expected_parses: 6, + }, + ]; + for test_case in test_cases { + let mut parses: usize = 0; + let result = probe_construct(LocalParser::Postgres, &PARENTHESES, &mut |sql| { + parses += 1; + by_length( + test_case.threshold, + &test_case.deep, + &test_case.shallow, + sql, + ) + }); + assert_eq!( + (result.failed_at, parses), + (test_case.expected_failed_at, test_case.expected_parses), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_parser_without_a_syntax_when_listing_constructs_then_only_its_own_are_probed() { + let test_cases = [ + ConstructsTestCase { + description: "Snowflake has ARRAY( types to mix with [] suffixes", + parser: LocalParser::Snowflake, + construct: "type_constructor_and_suffix", + expected_probed: true, + }, + ConstructsTestCase { + description: "Postgres has no type constructor", + parser: LocalParser::Postgres, + construct: "type_constructor_and_suffix", + expected_probed: false, + }, + ConstructsTestCase { + description: "every parser nests FROM parentheses", + parser: LocalParser::Postgres, + construct: "from_parentheses", + expected_probed: true, + }, + ]; + for test_case in test_cases { + let probed: bool = probed_constructs(test_case.parser) + .iter() + .any(|construct| construct.name == test_case.construct); + assert_eq!( + probed, test_case.expected_probed, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests/test_types.rs b/crates/sqlquack-syntax-coverage/src/depth/tests/test_types.rs new file mode 100644 index 000000000..2dd536158 --- /dev/null +++ b/crates/sqlquack-syntax-coverage/src/depth/tests/test_types.rs @@ -0,0 +1,31 @@ +use crate::check::models::{LocalFailure, LocalVerdict}; +use crate::depth::models::KnownFailure; +use crate::formats::models::LocalParser; + +/// A parser giving `deep` past `threshold` bytes of text and `shallow` below, and what the probe finds. +pub(crate) struct ProbeTestCase { + pub(crate) description: &'static str, + pub(crate) threshold: usize, + pub(crate) deep: LocalVerdict, + pub(crate) shallow: LocalVerdict, + pub(crate) expected_failed_at: Option<(usize, LocalFailure)>, + /// How many ladder depths the probe parses before it stops. + pub(crate) expected_parses: usize, +} + +/// A parser, a construct, and whether the probe runs that construct in that parser. +pub(crate) struct ConstructsTestCase { + pub(crate) description: &'static str, + pub(crate) parser: LocalParser, + pub(crate) construct: &'static str, + pub(crate) expected_probed: bool, +} + +/// Allowlist entries, and the `parser construct` each report list must name. +pub(crate) struct AllowlistTestCase { + pub(crate) description: &'static str, + pub(crate) entries: Vec, + pub(crate) expected_unlisted: Vec<&'static str>, + pub(crate) expected_stale: Vec<&'static str>, + pub(crate) expected_problems: Vec<&'static str>, +} diff --git a/crates/sqlquack-syntax-coverage/src/lib.rs b/crates/sqlquack-syntax-coverage/src/lib.rs index 567a38017..4b0dca70d 100644 --- a/crates/sqlquack-syntax-coverage/src/lib.rs +++ b/crates/sqlquack-syntax-coverage/src/lib.rs @@ -4,6 +4,7 @@ pub mod check; pub mod cli; +pub mod depth; pub mod formats; pub mod report; pub mod results; From f2f6250ce199dc887e11f75c48badece70eb48a4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 18:41:43 +0000 Subject: [PATCH 12/22] coverage: fail the depth probe on a lost result; probe casts nesting array types A probe thread that ends without a result no longer drops its construct silently: the run reports a problem unless every construct gave one. A new construct nests CAST( levels, each to INT with a run of [], so the combined type depth is probed in every parser, including Postgres. --- coverage/deep-nesting.toml | 16 ++++++++++++++++ .../src/cli/_helpers/depth_command.rs | 12 +++++++++++- .../src/depth/_helpers/constructs.rs | 15 +++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/coverage/deep-nesting.toml b/coverage/deep-nesting.toml index 7350039b2..5666640e2 100644 --- a/coverage/deep-nesting.toml +++ b/coverage/deep-nesting.toml @@ -209,6 +209,14 @@ depth = 100000 failure = "crash" owner = "sqlquack-syntax: shared tree serialisation and drop" +[[known_failures]] +parser = "postgres" +construct = "cast_and_type_suffix" +profile = "gate" +depth = 100000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + [[known_failures]] parser = "snowflake" construct = "type_suffix" @@ -432,3 +440,11 @@ profile = "dev" depth = 3000 failure = "crash" owner = "sqlquack-syntax: shared tree serialisation and drop" + +[[known_failures]] +parser = "postgres" +construct = "cast_and_type_suffix" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-postgres" diff --git a/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs b/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs index d67ef7aad..daa34403f 100644 --- a/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs +++ b/crates/sqlquack-syntax-coverage/src/cli/_helpers/depth_command.rs @@ -96,7 +96,17 @@ pub(crate) fn depth_probe( })?; let started: Instant = Instant::now(); let results: Vec = probe_all(&binary, jobs); - let report: DepthReport = compare_allowlist(&results, &known, profile); + let mut report: DepthReport = compare_allowlist(&results, &known, profile); + let expected: usize = PROBED_PARSERS + .iter() + .map(|&parser| probed_constructs(parser).len()) + .sum(); + if results.len() != expected { + report.problems.push(format!( + "{} of {expected} constructs gave no result; a probe thread failed", + expected - results.len().min(expected) + )); + } let failing: usize = results .iter() .filter(|result| result.failed_at.is_some()) diff --git a/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs index aa1db3f81..fb8f460fc 100644 --- a/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs +++ b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs @@ -36,6 +36,17 @@ fn mixed_type(parser: LocalParser, depth: usize) -> Option { } } +/// Casts nested about `depth / MIXED_SUFFIX_RUN` deep, each to a type with a run of `[]`. +fn cast_with_suffix(depth: usize) -> String { + let levels: usize = (depth / MIXED_SUFFIX_RUN).max(1); + let cast: String = format!(" AS INT{})", "[]".repeat(MIXED_SUFFIX_RUN)); + format!( + "SELECT {}NULL{}", + "CAST(".repeat(levels), + cast.repeat(levels) + ) +} + /// The probed constructs, in report order. pub(crate) fn constructs() -> Vec { vec![ @@ -179,6 +190,10 @@ pub(crate) fn constructs() -> Vec { mixed_type(parser, depth).map(|kind| format!("SELECT CAST(NULL AS {kind})")) }, }, + Construct { + name: "cast_and_type_suffix", + text: |_, depth| Some(cast_with_suffix(depth)), + }, Construct { name: "flat_plus", text: |_, depth| Some(format!("SELECT {}", chain("1", " + ", depth))), From 99e8a2145573f79caa1ee95b9cae513ffdf30849 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 20:38:59 +0000 Subject: [PATCH 13/22] coverage: probe BigQuery and chains nested in a chain's first operand The depth probe now links the BigQuery parser and nests chains of casts, additions, OR, UNION ALL and pipe operators inside the first operand of another chain, 60 links per parenthesised level, plus one flat UNION ALL chain inside a FROM subquery. A per-chain link count misses these trees. With the shared tree cloned, compared and serialised on a grown stack, the Snowflake parser has no known failure left, in either profile. The Postgres and BigQuery entries are their parsers' own recursion, which their depth limits remove when they land. --- coverage/deep-nesting.toml | 384 ++++++++++++++---- .../src/depth/_helpers/constructs.rs | 54 ++- .../src/depth/constants.rs | 8 +- .../src/depth/tests/test_probe_construct.rs | 12 + 4 files changed, 382 insertions(+), 76 deletions(-) diff --git a/coverage/deep-nesting.toml b/coverage/deep-nesting.toml index 5666640e2..c866e8233 100644 --- a/coverage/deep-nesting.toml +++ b/coverage/deep-nesting.toml @@ -9,38 +9,6 @@ # `depth` is the shallowest ladder depth that fails; `profile` is the Cargo profile probed: `gate` # in check-ci, `dev` in `make depth-probe-debug`. -[[known_failures]] -parser = "snowflake" -construct = "type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" - -[[known_failures]] -parser = "snowflake" -construct = "type_suffix_sized" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" - -[[known_failures]] -parser = "snowflake" -construct = "column_type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" - -[[known_failures]] -parser = "snowflake" -construct = "flat_plus" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" - [[known_failures]] parser = "postgres" construct = "from_parentheses" @@ -187,7 +155,7 @@ owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_plus" +construct = "cast_and_type_suffix" profile = "gate" depth = 100000 failure = "crash" @@ -195,67 +163,179 @@ owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_or" +construct = "nested_cast_chain" profile = "gate" depth = 100000 failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" +owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_union_all" +construct = "nested_plus_chain" profile = "gate" depth = 100000 failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" +owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "cast_and_type_suffix" +construct = "nested_or_chain" profile = "gate" depth = 100000 failure = "crash" owner = "sqlquack-syntax-postgres" [[known_failures]] -parser = "snowflake" -construct = "type_suffix" -profile = "dev" -depth = 1000 +parser = "postgres" +construct = "nested_union_all" +profile = "gate" +depth = 10000 failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" +owner = "sqlquack-syntax-postgres" [[known_failures]] -parser = "snowflake" -construct = "type_suffix_sized" -profile = "dev" -depth = 1000 +parser = "bigquery" +construct = "from_parentheses" +profile = "gate" +depth = 200 failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" +owner = "sqlquack-syntax-bigquery" [[known_failures]] -parser = "snowflake" -construct = "column_type_suffix" -profile = "dev" -depth = 1000 +parser = "bigquery" +construct = "from_parenthesised_join" +profile = "gate" +depth = 200 failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" +owner = "sqlquack-syntax-bigquery" [[known_failures]] -parser = "snowflake" -construct = "type_constructor_and_suffix" -profile = "dev" +parser = "bigquery" +construct = "from_subquery" +profile = "gate" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "scalar_subquery" +profile = "gate" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "exists_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "in_subquery" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_query" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_set_operation" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_expression" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "case" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "function_arguments" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "cast" +profile = "gate" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "subscript" +profile = "gate" depth = 1000 failure = "crash" -owner = "sqlquack-syntax: Snowflake parser" +owner = "sqlquack-syntax-bigquery" [[known_failures]] -parser = "snowflake" -construct = "flat_plus" -profile = "dev" +parser = "bigquery" +construct = "cast_and_type_suffix" +profile = "gate" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_plus_chain" +profile = "gate" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_or_chain" +profile = "gate" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_union_all" +profile = "gate" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_pipe_chain" +profile = "gate" depth = 10000 failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" +owner = "sqlquack-syntax-bigquery" [[known_failures]] parser = "postgres" @@ -389,7 +469,7 @@ owner = "sqlquack-syntax-postgres" parser = "postgres" construct = "type_suffix" profile = "dev" -depth = 1000 +depth = 10000 failure = "crash" owner = "sqlquack-syntax-postgres" @@ -397,7 +477,7 @@ owner = "sqlquack-syntax-postgres" parser = "postgres" construct = "type_suffix_sized" profile = "dev" -depth = 1000 +depth = 10000 failure = "crash" owner = "sqlquack-syntax-postgres" @@ -405,7 +485,7 @@ owner = "sqlquack-syntax-postgres" parser = "postgres" construct = "double_colon_type_suffix" profile = "dev" -depth = 1000 +depth = 10000 failure = "crash" owner = "sqlquack-syntax-postgres" @@ -413,13 +493,13 @@ owner = "sqlquack-syntax-postgres" parser = "postgres" construct = "column_type_suffix" profile = "dev" -depth = 1000 +depth = 10000 failure = "crash" owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_plus" +construct = "cast_and_type_suffix" profile = "dev" depth = 10000 failure = "crash" @@ -427,24 +507,184 @@ owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_or" +construct = "nested_cast_chain" profile = "dev" depth = 10000 failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" +owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "flat_union_all" +construct = "nested_plus_chain" profile = "dev" -depth = 3000 +depth = 10000 failure = "crash" -owner = "sqlquack-syntax: shared tree serialisation and drop" +owner = "sqlquack-syntax-postgres" [[known_failures]] parser = "postgres" -construct = "cast_and_type_suffix" +construct = "nested_or_chain" profile = "dev" depth = 10000 failure = "crash" owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "postgres" +construct = "nested_union_all" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-postgres" + +[[known_failures]] +parser = "bigquery" +construct = "from_parentheses" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "from_parenthesised_join" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "from_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "scalar_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "exists_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "in_subquery" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_query" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_set_operation" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_with" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "parenthesised_expression" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "case" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "function_arguments" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "cast" +profile = "dev" +depth = 50 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "subscript" +profile = "dev" +depth = 200 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "cast_and_type_suffix" +profile = "dev" +depth = 3000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_plus_chain" +profile = "dev" +depth = 3000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_or_chain" +profile = "dev" +depth = 3000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_union_all" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" + +[[known_failures]] +parser = "bigquery" +construct = "nested_pipe_chain" +profile = "dev" +depth = 10000 +failure = "crash" +owner = "sqlquack-syntax-bigquery" diff --git a/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs index fb8f460fc..4c79ea085 100644 --- a/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs +++ b/crates/sqlquack-syntax-coverage/src/depth/_helpers/constructs.rs @@ -1,6 +1,6 @@ -//! Every construct the probe nests: recursive paths through queries, expressions and types, and flat chains. +//! Every construct the probe nests: recursive paths, flat chains, and chains nested in a chain's first operand. -use crate::depth::constants::MIXED_SUFFIX_RUN; +use crate::depth::constants::{MIXED_SUFFIX_RUN, NESTED_CHAIN_LINKS}; use crate::depth::models::Construct; use crate::formats::models::LocalParser; @@ -14,11 +14,19 @@ fn chain(term: &str, separator: &str, depth: usize) -> String { vec![term; depth].join(separator) } +/// A parenthesised chain of `link` after `inner`, nested in the first operand of the next, `depth` links in all. +fn nested_chain(inner: &str, link: &str, depth: usize) -> String { + let levels: usize = (depth / NESTED_CHAIN_LINKS).max(1); + let close: String = format!("{})", link.repeat(NESTED_CHAIN_LINKS)); + nest("(", inner, &close, levels) +} + /// A type built by the parser's own nested type constructor, where it has one. fn constructed_type(parser: LocalParser, depth: usize) -> Option { match parser { LocalParser::Snowflake => Some(nest("ARRAY(", "INT", ")", depth)), - LocalParser::Postgres | LocalParser::BigQuery | LocalParser::None => None, + LocalParser::BigQuery => Some(nest("STRUCT", depth)), + LocalParser::Postgres | LocalParser::None => None, } } @@ -211,6 +219,46 @@ pub(crate) fn constructs() -> Vec { name: "flat_union_all", text: |_, depth| Some(chain("SELECT 1", " UNION ALL ", depth)), }, + Construct { + name: "subquery_flat_union_all", + text: |_, depth| { + Some(format!( + "SELECT * FROM ({}) s", + chain("SELECT 1", " UNION ALL ", depth) + )) + }, + }, + Construct { + name: "nested_cast_chain", + text: |parser, depth| match parser { + LocalParser::BigQuery => None, + _ => Some(format!("SELECT {}", nested_chain("1", "::INT", depth))), + }, + }, + Construct { + name: "nested_plus_chain", + text: |_, depth| Some(format!("SELECT {}", nested_chain("1", " + 1", depth))), + }, + Construct { + name: "nested_or_chain", + text: |_, depth| { + Some(format!( + "SELECT {}", + nested_chain("FALSE", " OR FALSE", depth) + )) + }, + }, + Construct { + name: "nested_union_all", + text: |_, depth| Some(nested_chain("SELECT 1", " UNION ALL SELECT 1", depth)), + }, + Construct { + name: "nested_pipe_chain", + text: |parser, depth| match parser { + LocalParser::BigQuery => Some(nested_chain("FROM t", " |> WHERE TRUE", depth)), + _ => None, + }, + }, Construct { name: "flat_join", text: |_, depth| { diff --git a/crates/sqlquack-syntax-coverage/src/depth/constants.rs b/crates/sqlquack-syntax-coverage/src/depth/constants.rs index 37562f7d3..45d2736fc 100644 --- a/crates/sqlquack-syntax-coverage/src/depth/constants.rs +++ b/crates/sqlquack-syntax-coverage/src/depth/constants.rs @@ -14,5 +14,11 @@ pub const PROBE_BOUND: Duration = Duration::from_secs(15); pub const PROBE_STACK_BYTES: usize = 2 * 1024 * 1024; /// How many runs of `[]` a mixed type nests between its constructor levels. pub const MIXED_SUFFIX_RUN: usize = 60; +/// How many links each parenthesised level of a nested chain holds. +pub const NESTED_CHAIN_LINKS: usize = 60; /// Every native parser this checker links, each probed in turn. -pub const PROBED_PARSERS: [LocalParser; 2] = [LocalParser::Snowflake, LocalParser::Postgres]; +pub const PROBED_PARSERS: [LocalParser; 3] = [ + LocalParser::Snowflake, + LocalParser::Postgres, + LocalParser::BigQuery, +]; diff --git a/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs b/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs index 3768f2b50..48b81020e 100644 --- a/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs +++ b/crates/sqlquack-syntax-coverage/src/depth/tests/test_probe_construct.rs @@ -84,6 +84,18 @@ fn given_a_parser_without_a_syntax_when_listing_constructs_then_only_its_own_are construct: "type_constructor_and_suffix", expected_probed: false, }, + ConstructsTestCase { + description: "BigQuery chains pipe operators", + parser: LocalParser::BigQuery, + construct: "nested_pipe_chain", + expected_probed: true, + }, + ConstructsTestCase { + description: "Postgres has no pipe operator", + parser: LocalParser::Postgres, + construct: "nested_pipe_chain", + expected_probed: false, + }, ConstructsTestCase { description: "every parser nests FROM parentheses", parser: LocalParser::Postgres, From 0388b67a61110e2b9c7d18d8f9415dafd21fa3b4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:17:40 +0000 Subject: [PATCH 14/22] Postgres parser keeps only failures and non-overlapping JSON_OBJECT trees in its memo A parenthesized query's successful parse is no longer copied into the memo, so nested queries no longer hold a copy of their subtree at every level. Keeping a JSON_OBJECT argument tree drops the kept trees nested inside it, so kept trees never overlap, and their total nodes are bounded per statement. --- crates/sqlquack-syntax-postgres/README.md | 11 +++-- .../src/parse/_helpers/tree/cursor.rs | 47 ++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/crates/sqlquack-syntax-postgres/README.md b/crates/sqlquack-syntax-postgres/README.md index 88c20d99a..2b976c17e 100644 --- a/crates/sqlquack-syntax-postgres/README.md +++ b/crates/sqlquack-syntax-postgres/README.md @@ -28,10 +28,13 @@ Nesting is bounded so that no input exhausts the caller's stack or takes exponen - Local parser limit, tree depth: PostgreSQL 18.6 accepts nested casts to about 13,200 levels (132 levels of 99 casts each) before its own 54001, so a statement 8,000 to 13,200 levels deep is refused here and accepted by the engine. -- A parenthesized query, and `JSON_OBJECT`'s arguments, are parsed once from each token and the - outcome replayed, so alternatives that each read the same nested text stay linear. A replayed - tree is cloned recursively, so it is cloned on a stack grown to its depth, and a tree past the - 8,000-level limit below is refused before it is kept. +- A parenthesized query, and `JSON_OBJECT`'s arguments, that fail to parse from a token have the + failure replayed there. `JSON_OBJECT`'s arguments also keep their tree, because its readings each + read the nested arguments; keeping an outer one drops those kept inside it, so the kept trees + never overlap. A parenthesized query's tree is not kept, since keeping a copy at every level grows + with depth times size. Kept trees are cloned on a stack grown to their depth, and together may + hold at most 65,536 nodes plus 32 per input token; an input past that is refused with the same + error. This is a local parser limit: PostgreSQL accepts such inputs. ## Gaps diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs index f9029423c..7a91466a6 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs @@ -1,6 +1,6 @@ //! The parser's state: the tokens, where it is, and how tokens read as names. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use sqlquack_syntax::expressions::names::models::Identifier; use sqlquack_syntax::{Expression, ExpressionWalk}; @@ -34,6 +34,12 @@ const CLONE_LEVEL_BYTES_DEBUG: usize = 8 * 1024; #[cfg(feature = "stacker")] const CLONE_STACK_MARGIN: usize = 1024 * 1024; +/// Remembered nodes allowed per input token, beyond `MEMO_BASE_NODES`, before the input is refused. +const MEMO_NODES_PER_TOKEN: usize = 32; + +/// Remembered nodes allowed whatever the input's length. +const MEMO_BASE_NODES: usize = 64 * 1024; + /// PostgreSQL's message for 54001, which recursion and chains past their limits both raise. const STACK_DEPTH_EXCEEDED: &str = "stack depth limit exceeded"; @@ -45,6 +51,7 @@ pub(crate) enum Remembered { } /// How a remembered construct's parse ended: its result, where it stopped, and the gaps it found. +/// Only failures and `JSON_OBJECT` arguments are kept, so the copies stay bounded. struct Outcome { result: Parsing, reached: usize, @@ -60,6 +67,10 @@ pub(crate) struct Parser<'sql> { too_deep: Option, /// Constructs already parsed from a token, replayed when another alternative reaches them. remembered: HashMap<(Remembered, usize), Outcome>, + /// The nodes of every tree kept in `remembered`, bounded by the memo budget. + remembered_nodes: usize, + /// The start token and node count of every `JSON_OBJECT` argument tree kept in `remembered`. + kept_trees: BTreeMap, pub(crate) gaps: Vec, /// Whether the statement being parsed holds a gap, so it becomes a `Command`. pub(crate) opaque: bool, @@ -141,6 +152,8 @@ impl<'sql> Parser<'sql> { depth: 0, too_deep: None, remembered: HashMap::new(), + remembered_nodes: 0, + kept_trees: BTreeMap::new(), gaps: Vec::new(), opaque: false, } @@ -441,8 +454,23 @@ impl<'sql> Parser<'sql> { Ok(tree) if tree.tree_depth() > MAX_TREE_DEPTH => Err(self.refuse_depth()), result => result, }; + let kept: Parsing = match (&result, construct) { + (Err(error), _) => Err(error.clone()), + (Ok(_), Remembered::ParenthesizedQuery) => return result, + (Ok(tree), Remembered::JsonObjectArguments) => { + self.forget_trees_within(key.1, self.at); + let nodes: usize = tree.dfs().count(); + self.kept_trees.insert(key.1, nodes); + self.remembered_nodes += nodes; + let budget: usize = MEMO_BASE_NODES + MEMO_NODES_PER_TOKEN * self.tokens.len(); + if self.remembered_nodes > budget { + return Err(self.refuse_depth()); + } + copied(&result) + } + }; let outcome: Outcome = Outcome { - result: copied(&result), + result: kept, reached: self.at, gaps: self.gaps[gaps.min(self.gaps.len())..].to_vec(), }; @@ -450,6 +478,21 @@ impl<'sql> Parser<'sql> { result } + /// Drops kept argument trees strictly inside `start..end`; only that span's replay reached them. + fn forget_trees_within(&mut self, start: usize, end: usize) { + let inner: Vec<(usize, usize)> = self + .kept_trees + .range(start + 1..end) + .map(|(&at, &nodes)| (at, nodes)) + .collect(); + for (at, nodes) in inner { + self.kept_trees.remove(&at); + self.remembered + .remove(&(Remembered::JsonObjectArguments, at)); + self.remembered_nodes -= nodes; + } + } + /// `result`, or the depth limit's error if any path went past it. pub(crate) fn unless_too_deep(&self, result: Parsing) -> Parsing { match &self.too_deep { From 8dacf52fd28da35601c862e36faedfd217913b47 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 21:21:25 +0000 Subject: [PATCH 15/22] fix: drop every node a parser chains in a loop without recursion A BigQuery pipe chain that ends in a syntax error is dropped on the error path before any depth guard runs, and PipeOperator's derived drop recursed once per step, so 20,000 steps aborted a 2 MiB debug thread. drop_iteratively! now covers PipeOperator and the other nodes a parser wraps around its own result in a loop: AT TIME ZONE, ** (BinaryFunc), XOR, the Snowflake ->> flow operator, model attributes, .: casts, array slices and method calls. Paren and Annotated stay derived: the parser moves out of them, the token guard bounds parentheses, and an annotation wraps one operand once. The BigQuery parser's with_ctes moved the input out of a PipeOperator, which a Drop impl forbids; it now walks to the chain's first input and replaces it in place, without recursion. A test builds a 100,000-step chain of each newly covered kind and drops it on a 2 MiB thread; it aborts without this change. --- .../src/parse/_helpers/commands/queries.rs | 8 +- .../src/expressions/tree/_helpers/drops.rs | 20 ++- .../src/guard/tests/test_ast_guard.rs | 136 ++++++++++++++++++ .../src/guard/tests/test_types.rs | 9 ++ 4 files changed, 171 insertions(+), 2 deletions(-) diff --git a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/queries.rs b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/queries.rs index f27a3e66d..27ca4b6bf 100644 --- a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/queries.rs +++ b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/queries.rs @@ -2,6 +2,7 @@ use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Identifier; +use sqlquack_syntax::expressions::names::models::Null; use sqlquack_syntax::expressions::operators::models::Alias; use sqlquack_syntax::expressions::queries::models::{ ClusterBy, Cte, DistributeBy, From, GroupBy, Having, Limit, NamedWindow, OrderBy, Ordered, @@ -237,7 +238,12 @@ fn reorganized(body: Expression, organization: Organization) -> Expression { /// `body` with `with` as its common table expressions, wrapped when it has its own already. pub(crate) fn with_ctes(body: Expression, with: With) -> Expression { if let Expression::PipeOperator(mut node) = body { - node.this = with_ctes(node.this, with); + let mut input = &mut node.this; + while let Expression::PipeOperator(inner) = input { + input = &mut inner.this; + } + let first = std::mem::replace(input, Expression::Null(Null)); + *input = with_ctes(first, with); return Expression::PipeOperator(node); } let has_with: bool = match &body { diff --git a/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs b/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs index a771d0fe5..416b1ed15 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs @@ -1,6 +1,10 @@ -//! Iterative drops for nodes that the parser chains in a loop, such as `x::INT::INT` or `v[1][1]`. +//! Iterative drops for nodes a parser chains in a loop, such as `x::INT::INT`, `v[1][1]` or `q |> WHERE a |> WHERE b`. +use crate::expressions::functions::models::JSONCast; +use crate::expressions::functions::models::Operator; +use crate::expressions::functions::models::Xor; use crate::expressions::operators::models::Alias; +use crate::expressions::operators::models::ArraySlice; use crate::expressions::operators::models::Between; use crate::expressions::operators::models::BinaryOp; use crate::expressions::operators::models::Cast; @@ -10,10 +14,15 @@ use crate::expressions::operators::models::In; use crate::expressions::operators::models::IsNull; use crate::expressions::operators::models::IsTrueFalse; use crate::expressions::operators::models::LikeOp; +use crate::expressions::operators::models::MethodCall; use crate::expressions::operators::models::Subscript; use crate::expressions::operators::models::UnaryOp; +use crate::expressions::queries::models::PipeOperator; use crate::expressions::tree::models::Expression; +use crate::expressions::typed_functions::models::AtTimeZone; +use crate::expressions::typed_functions::models::BinaryFunc; use crate::expressions::typed_functions::models::JsonExtractFunc; +use crate::expressions::typed_functions::models::ModelAttribute; use crate::traversal::types::AstNode; /// Frees a node's expression descendants with a worklist, so dropping a long chain cannot overflow the stack. @@ -39,7 +48,10 @@ macro_rules! drop_iteratively { drop_iteratively!( Alias, + ArraySlice, + AtTimeZone, Between, + BinaryFunc, BinaryOp, Cast, CollationExpr, @@ -47,8 +59,14 @@ drop_iteratively!( In, IsNull, IsTrueFalse, + JSONCast, JsonExtractFunc, LikeOp, + MethodCall, + ModelAttribute, + Operator, + PipeOperator, Subscript, UnaryOp, + Xor, ); diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index c394c98b1..01eef6243 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -1,12 +1,26 @@ +use crate::expressions::functions::models::JSONCast; +use crate::expressions::functions::models::Operator; +use crate::expressions::functions::models::Xor; +use crate::expressions::names::models::Identifier; +use crate::expressions::operators::models::ArraySlice; +use crate::expressions::operators::models::MethodCall; +use crate::expressions::queries::models::PipeOperator; +use crate::expressions::tree::models::Expression; +use crate::expressions::typed_functions::models::AtTimeZone; +use crate::expressions::typed_functions::models::BinaryFunc; +use crate::expressions::typed_functions::models::ModelAttribute; +use crate::expressions::types::models::DataType; use crate::guard::tests::helpers::constructor_levels; use crate::guard::tests::helpers::debug_length; use crate::guard::tests::helpers::deep_trees; use crate::guard::tests::helpers::guarded_parse; use crate::guard::tests::helpers::guarded_parse_in; use crate::guard::tests::test_types::AstDepthTestCase; +use crate::guard::tests::test_types::ChainStepTestCase; use crate::guard::tests::test_types::TreeOperationTestCase; use crate::guard::tests::test_types::TypeDepthTestCase; use crate::guard::tests::test_types::TYPE_DEPTHS; +use crate::traversal::types::ExpressionWalk; #[test] fn given_a_long_flat_operator_chain_when_parsing_then_only_real_nesting_counts_as_depth() { @@ -328,3 +342,125 @@ fn given_long_chains_and_deep_nesting_on_a_2_mb_thread_when_cloning_comparing_fo ); } } + +#[test] +fn given_a_100_000_step_chain_of_a_node_parsers_build_in_a_loop_when_dropping_on_a_2_mb_thread_then_it_does_not_overflow( +) { + let test_cases = [ + ChainStepTestCase { + description: "BigQuery pipe operators", + step: |input| { + Expression::PipeOperator(Box::new(PipeOperator { + this: input, + expression: Expression::number(1), + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "AT TIME ZONE", + step: |input| { + Expression::AtTimeZone(Box::new(AtTimeZone { + this: input, + zone: Expression::number(1), + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "** powers", + step: |input| { + Expression::Power(Box::new(BinaryFunc { + this: input, + expression: Expression::number(1), + original_name: None, + inferred_type: None, + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "XOR", + step: |input| { + Expression::Xor(Box::new(Xor { + this: Some(Box::new(input)), + expression: Some(Box::new(Expression::number(1))), + expressions: Vec::new(), + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "Snowflake ->> flow operators", + step: |input| { + Expression::Operator(Box::new(Operator { + this: Box::new(input), + operator: None, + expression: Box::new(Expression::number(1)), + comments: Vec::new(), + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "model attributes", + step: |input| { + Expression::ModelAttribute(Box::new(ModelAttribute { + this: Box::new(input), + expression: Box::new(Expression::number(1)), + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: ".: casts", + step: |input| { + Expression::JSONCast(Box::new(JSONCast { + this: Box::new(input), + to: DataType::Json, + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "array slices", + step: |input| { + Expression::ArraySlice(Box::new(ArraySlice { + this: input, + start: Some(Expression::number(1)), + end: None, + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "method calls", + step: |input| { + Expression::MethodCall(Box::new(MethodCall { + this: input, + method: Identifier::new("f"), + args: vec![Expression::number(1)], + })) + }, + expected_depth: 100_000, + }, + ]; + for test_case in test_cases { + let step = test_case.step; + let depth = std::thread::Builder::new() + .stack_size(2 << 20) + .spawn(move || { + let mut chain = Expression::number(1); + for _ in 0..100_000 { + chain = step(chain); + } + let depth = chain.tree_depth(); + drop(chain); + depth + }) + .expect("the thread starts") + .join() + .expect("the chain drops"); + assert_eq!(depth, test_case.expected_depth, "{}", test_case.description); + } +} diff --git a/crates/sqlquack-syntax/src/guard/tests/test_types.rs b/crates/sqlquack-syntax/src/guard/tests/test_types.rs index 664a77430..dfe0b638a 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_types.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_types.rs @@ -1,5 +1,6 @@ use serde_json::error::Category; +use crate::expressions::tree::models::Expression; use crate::guard::models::ComplexityGuardOptions; pub(crate) struct GuardJsonTestCase { @@ -57,3 +58,11 @@ pub(crate) struct TreeOperationTestCase { pub(crate) operation: fn(Vec) -> bool, pub(crate) expected_completed: bool, } + +pub(crate) struct ChainStepTestCase { + pub(crate) description: &'static str, + /// Wraps the chain so far in one more node of the kind under test. + pub(crate) step: fn(Expression) -> Expression, + /// The chain's depth after 100,000 steps, which the drop must survive. + pub(crate) expected_depth: usize, +} From 5e34e9a7d9800169c9b5ffe4a5f1565f69116435 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:36:27 +0000 Subject: [PATCH 16/22] coverage: drop the Postgres deep-nesting entries the depth guard fixes --- coverage/deep-nesting.toml | 384 ------------------------------------- 1 file changed, 384 deletions(-) diff --git a/coverage/deep-nesting.toml b/coverage/deep-nesting.toml index c866e8233..daf394d96 100644 --- a/coverage/deep-nesting.toml +++ b/coverage/deep-nesting.toml @@ -9,190 +9,6 @@ # `depth` is the shallowest ladder depth that fails; `profile` is the Cargo profile probed: `gate` # in check-ci, `dev` in `make depth-probe-debug`. -[[known_failures]] -parser = "postgres" -construct = "from_parentheses" -profile = "gate" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "from_parenthesised_join" -profile = "gate" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "from_subquery" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "scalar_subquery" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "exists_subquery" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "in_subquery" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_query" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_set_operation" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_with" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_expression" -profile = "gate" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "case" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "function_arguments" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "cast" -profile = "gate" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "subscript" -profile = "gate" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "type_suffix_sized" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "double_colon_type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "column_type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "cast_and_type_suffix" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_cast_chain" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_plus_chain" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_or_chain" -profile = "gate" -depth = 100000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_union_all" -profile = "gate" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - [[known_failures]] parser = "bigquery" construct = "from_parentheses" @@ -337,206 +153,6 @@ depth = 10000 failure = "crash" owner = "sqlquack-syntax-bigquery" -[[known_failures]] -parser = "postgres" -construct = "from_parentheses" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "from_parenthesised_join" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "from_subquery" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "scalar_subquery" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "exists_subquery" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "in_subquery" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_query" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_set_operation" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_with" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "parenthesised_expression" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "unary_minus" -profile = "dev" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "not" -profile = "dev" -depth = 1000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "case" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "function_arguments" -profile = "dev" -depth = 50 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "cast" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "subscript" -profile = "dev" -depth = 200 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "type_suffix" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "type_suffix_sized" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "double_colon_type_suffix" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "column_type_suffix" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "cast_and_type_suffix" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_cast_chain" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_plus_chain" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_or_chain" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - -[[known_failures]] -parser = "postgres" -construct = "nested_union_all" -profile = "dev" -depth = 10000 -failure = "crash" -owner = "sqlquack-syntax-postgres" - [[known_failures]] parser = "bigquery" construct = "from_parentheses" From ffbd657df7f32c4bdef47864a454b0aa5d014f87 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:44:36 +0000 Subject: [PATCH 17/22] fix: drop TABLESAMPLE and MATCH_RECOGNIZE chains without recursion The BigQuery parser wraps the relation in a new TableSample or MatchRecognize once per repeated clause after FROM, through a helper constructor, so FROM t followed by 20,000 TABLESAMPLE clauses built a chain whose derived drop aborted a 2 MiB debug thread, including on the syntax error path. Both now drop iteratively, and the 100,000-step drop test covers them; it aborts without this change. --- .../src/expressions/tree/_helpers/drops.rs | 4 ++ .../src/guard/tests/test_ast_guard.rs | 40 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs b/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs index 416b1ed15..4de3ab977 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs @@ -17,7 +17,9 @@ use crate::expressions::operators::models::LikeOp; use crate::expressions::operators::models::MethodCall; use crate::expressions::operators::models::Subscript; use crate::expressions::operators::models::UnaryOp; +use crate::expressions::queries::models::MatchRecognize; use crate::expressions::queries::models::PipeOperator; +use crate::expressions::statements::models::TableSample; use crate::expressions::tree::models::Expression; use crate::expressions::typed_functions::models::AtTimeZone; use crate::expressions::typed_functions::models::BinaryFunc; @@ -62,11 +64,13 @@ drop_iteratively!( JSONCast, JsonExtractFunc, LikeOp, + MatchRecognize, MethodCall, ModelAttribute, Operator, PipeOperator, Subscript, + TableSample, UnaryOp, Xor, ); diff --git a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs index 01eef6243..f55cdf06e 100644 --- a/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs +++ b/crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs @@ -4,7 +4,9 @@ use crate::expressions::functions::models::Xor; use crate::expressions::names::models::Identifier; use crate::expressions::operators::models::ArraySlice; use crate::expressions::operators::models::MethodCall; +use crate::expressions::queries::models::MatchRecognize; use crate::expressions::queries::models::PipeOperator; +use crate::expressions::statements::models::TableSample; use crate::expressions::tree::models::Expression; use crate::expressions::typed_functions::models::AtTimeZone; use crate::expressions::typed_functions::models::BinaryFunc; @@ -433,6 +435,44 @@ fn given_a_100_000_step_chain_of_a_node_parsers_build_in_a_loop_when_dropping_on }, expected_depth: 100_000, }, + ChainStepTestCase { + description: "TABLESAMPLE after TABLESAMPLE", + step: |input| { + Expression::TableSample(Box::new(TableSample { + this: Some(Box::new(input)), + sample: None, + expressions: Vec::new(), + method: None, + bucket_numerator: None, + bucket_denominator: None, + bucket_field: None, + percent: Some(Box::new(Expression::number(1))), + rows: None, + size: None, + seed: None, + })) + }, + expected_depth: 100_000, + }, + ChainStepTestCase { + description: "MATCH_RECOGNIZE after MATCH_RECOGNIZE", + step: |input| { + Expression::MatchRecognize(Box::new(MatchRecognize { + this: Some(Box::new(input)), + partition_by: None, + order_by: None, + measures: None, + rows: None, + after: None, + pattern: None, + define: None, + alias: None, + alias_explicit_as: false, + options: vec![Expression::number(1)], + })) + }, + expected_depth: 100_000, + }, ChainStepTestCase { description: "method calls", step: |input| { From aeba313e962dcf1d9603287537acca3177e052d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:46:14 +0000 Subject: [PATCH 18/22] feat(format): add sqlquack-format, a token-level SQL formatter, with Snowflake layout The formatter lays out SQL over SQLQuack's lossless tokens and parse tree, copying only whitespace and keyword case, and checks token identity and idempotency before it returns. It is gated on SQLGlot's formatting fixtures (MIT, see FORMAT_CORPUS_NOTICE.md) for Snowflake. --- Cargo.lock | 8 + Cargo.toml | 2 + THIRD_PARTY_NOTICES | 5 + crates/sqlquack-format/Cargo.toml | 17 + .../src/format/_helpers/guide.rs | 1053 ++++ .../src/format/_helpers/identifiers.rs | 256 + .../src/format/_helpers/lexemes.rs | 132 + .../src/format/_helpers/mod.rs | 9 + .../src/format/_helpers/pipeline.rs | 223 + .../src/format/_helpers/printed_layout.rs | 1066 ++++ .../src/format/_helpers/token_layout.rs | 597 ++ .../src/format/_helpers/touching.rs | 102 + .../sqlquack-format/src/format/constants.rs | 320 + .../src/format/main/format_sql.rs | 9 + crates/sqlquack-format/src/format/main/mod.rs | 3 + crates/sqlquack-format/src/format/mod.rs | 9 + crates/sqlquack-format/src/format/models.rs | 51 + crates/sqlquack-format/src/format/tests.rs | 8 + .../tests/fixtures/FORMAT_CORPUS_NOTICE.md | 32 + .../format/tests/fixtures/format_corpus.json | 5354 +++++++++++++++++ .../src/format/tests/helpers.rs | 163 + .../src/format/tests/test_format_corpus.rs | 131 + .../src/format/tests/test_format_sql.rs | 413 ++ .../src/format/tests/test_types.rs | 55 + crates/sqlquack-format/src/lib.rs | 8 + docs/architecture.md | 1 + 26 files changed, 10027 insertions(+) create mode 100644 crates/sqlquack-format/Cargo.toml create mode 100644 crates/sqlquack-format/src/format/_helpers/guide.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/identifiers.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/lexemes.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/mod.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/pipeline.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/printed_layout.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/token_layout.rs create mode 100644 crates/sqlquack-format/src/format/_helpers/touching.rs create mode 100644 crates/sqlquack-format/src/format/constants.rs create mode 100644 crates/sqlquack-format/src/format/main/format_sql.rs create mode 100644 crates/sqlquack-format/src/format/main/mod.rs create mode 100644 crates/sqlquack-format/src/format/mod.rs create mode 100644 crates/sqlquack-format/src/format/models.rs create mode 100644 crates/sqlquack-format/src/format/tests.rs create mode 100644 crates/sqlquack-format/src/format/tests/fixtures/FORMAT_CORPUS_NOTICE.md create mode 100644 crates/sqlquack-format/src/format/tests/fixtures/format_corpus.json create mode 100644 crates/sqlquack-format/src/format/tests/helpers.rs create mode 100644 crates/sqlquack-format/src/format/tests/test_format_corpus.rs create mode 100644 crates/sqlquack-format/src/format/tests/test_format_sql.rs create mode 100644 crates/sqlquack-format/src/format/tests/test_types.rs create mode 100644 crates/sqlquack-format/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 0d8fc7aca..542d6a3eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1408,6 +1408,14 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sqlquack-format" +version = "0.2.0" +dependencies = [ + "serde_json", + "sqlquack-syntax", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 9b67fd5d9..30eb6684c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/sqlquack-polyglot-check", "crates/sqlquack-syntax-coverage", "crates/sqlquack-syntax-postgres", + "crates/sqlquack-format", "crates/sqlquack-syntax-bigquery", "crates/sqlquack-python", ] @@ -58,6 +59,7 @@ sqlquack-compare = { path = "crates/sqlquack-compare" } sqlquack-core = { path = "crates/sqlquack-core" } sqlquack-duckdb = { path = "crates/sqlquack-duckdb", default-features = false } sqlquack-evidence = { path = "crates/sqlquack-evidence" } +sqlquack-format = { path = "crates/sqlquack-format" } sqlquack-policy = { path = "crates/sqlquack-policy" } sqlquack-rules = { path = "crates/sqlquack-rules" } sqlquack-semantics = { path = "crates/sqlquack-semantics" } diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 7c515c781..796addf9c 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -22,6 +22,11 @@ The referee corpus referee/corpora/sqlglot.json holds the query statements passe `validate_identity` and `validate_all` in tests/dialects/test_snowflake.py at the sqlglot commit its `origin` field names. Only Snowflake's syntax verdict on each statement is recorded. +The formatter corpus crates/sqlquack-format/src/format/tests/fixtures/format_corpus.json holds +SELECT and WITH queries from sqlglot's dialect identity test fixtures, as extracted by polyglot, +with the layouts SQLClean's formatter gives them. It is copied unchanged from SQLClean +(https://github.com/sqlstack-io/sqlclean), with its notice FORMAT_CORPUS_NOTICE.md beside it. + sqlglot is distributed under the following licence: MIT License diff --git a/crates/sqlquack-format/Cargo.toml b/crates/sqlquack-format/Cargo.toml new file mode 100644 index 000000000..ea8636dbb --- /dev/null +++ b/crates/sqlquack-format/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sqlquack-format" +version.workspace = true +description = "SQL layout that changes only the whitespace between tokens and the case of keywords." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +serde_json.workspace = true +sqlquack-syntax.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-format/src/format/_helpers/guide.rs b/crates/sqlquack-format/src/format/_helpers/guide.rs new file mode 100644 index 000000000..0a9499f83 --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/guide.rs @@ -0,0 +1,1053 @@ +//! The layout guide: the line breaks a parse-tree pretty printer puts between the authored tokens. + +use std::collections::HashMap; + +use sqlquack_syntax::Dialect; +use sqlquack_syntax::tokens::models::TokenType; + +use crate::format::_helpers::identifiers::ParsedNames; +use crate::format::_helpers::lexemes::{ + Lexeme, bracket_partners, ends_operand, is_keyword, is_unquoted_word, +}; +use crate::format::constants::{ + ALIAS_FOLLOWERS, CASE_BRANCHES, CLAUSE_KEYWORDS, DUCKDB_INLINE_CALLS, GROUPING_CALLS, + GUIDE_INDENT, GUIDE_WIDTH, JOIN_PREFIXES, PLACEHOLDER_WIDTH, POSTFIX_CAST_WIDTH, + POSTGRES_INLINE_CALLS, SAMPLE_ROW_KEYWORD, SEMANTIC_VIEW_CALL, SEMANTIC_VIEW_CLAUSES, + SET_OPERATOR_MODIFIERS, SET_OPERATORS, SNOWFLAKE_INLINE_CALLS, SPACED_BEFORE_PARENTHESIS, + STRUCT_TYPE_WORDS, TYPE_CONTINUATIONS, TYPED_LITERALS, +}; + +/// The guide's text and its tokens, with byte ranges in that text. +pub(crate) struct GuideLayout { + pub(crate) text: String, + pub(crate) tokens: Vec, + /// Token ranges whose authored text is kept byte for byte, such as select items without an alias. + pub(crate) verbatim: Vec<(usize, usize)>, + /// The number of select lists laid out. + pub(crate) selects: usize, +} + +/// Lay out `lexemes` with what the parse tree says about them in `names`. +pub(crate) fn layout_guide( + lexemes: &[Lexeme], + names: &ParsedNames, + dialect: Dialect, +) -> GuideLayout { + let mut guide = Guide::new(lexemes, names, dialect); + guide.statements(); + guide.render() +} + +struct Guide<'a> { + lexemes: &'a [Lexeme], + identifiers: &'a [bool], + /// Tokens the parse tree names as an alias. + aliases: &'a [bool], + /// Token ranges whose authored text is kept byte for byte. + verbatim: Vec<(usize, usize)>, + /// The number of select lists laid out. + selects: usize, + dialect: Dialect, + partners: Vec>, + /// The innermost bracket each token is inside. + enclosing: Vec>, + /// Tokens of a Snowflake stage reference such as `@stage/path/file.csv`. + in_stage: Vec, + /// The one-line spacing before each token. + base: Vec<&'static str>, + /// The laid-out whitespace before each token. + gaps: Vec, + /// Tokens the pretty printer leaves out, such as the `{fn ...}` escape. + dropped: Vec, + /// The width each token adds to a one-line rendering with literals replaced by placeholders. + widths: Vec, + /// Tokens inside a placeholder after its first token, which add no width. + covered: Vec, + /// Tokens a placeholder stands in for, which keep their authored case. + protected: Vec, +} + +fn newline(indent: usize) -> String { + format!("\n{}", " ".repeat(indent * GUIDE_INDENT)) +} + +fn sentinel_width(sequence: usize) -> usize { + PLACEHOLDER_WIDTH + sequence.to_string().len() +} + +impl<'a> Guide<'a> { + fn new(lexemes: &'a [Lexeme], names: &'a ParsedNames, dialect: Dialect) -> Self { + let count = lexemes.len(); + let mut enclosing: Vec> = vec![None; count]; + let mut open: Vec = Vec::new(); + for (index, lexeme) in lexemes.iter().enumerate() { + match lexeme.kind { + TokenType::LParen | TokenType::LBracket | TokenType::LBrace => { + enclosing[index] = open.last().copied(); + open.push(index); + } + TokenType::RParen | TokenType::RBracket | TokenType::RBrace => { + open.pop(); + enclosing[index] = open.last().copied(); + } + _ => enclosing[index] = open.last().copied(), + } + } + let mut guide = Self { + lexemes, + identifiers: &names.identifiers, + aliases: &names.aliases, + verbatim: Vec::new(), + selects: 0, + dialect, + partners: bracket_partners(lexemes), + enclosing, + in_stage: vec![false; count], + base: vec![""; count], + gaps: vec![String::new(); count], + dropped: vec![false; count], + widths: vec![0; count], + covered: vec![false; count], + protected: vec![false; count], + }; + guide.mark_stages(); + for index in 1..count { + guide.base[index] = guide.space(index); + guide.gaps[index] = guide.base[index].to_string(); + } + guide.measure(); + guide + } + + fn len(&self) -> usize { + self.lexemes.len() + } + + fn kind(&self, index: usize) -> Option { + self.lexemes.get(index).map(|lexeme| lexeme.kind) + } + + /// Whether token `index` is the keyword `word`, not an identifier spelled like it. + fn is(&self, index: usize, word: &str) -> bool { + self.lexemes + .get(index) + .is_some_and(|lexeme| lexeme.raw.eq_ignore_ascii_case(word)) + && !self.identifiers[index] + && !self.name_position(index) + } + + /// Whether token `index` is any of the keywords `words`. + fn is_any(&self, index: usize, words: &[&str]) -> bool { + words.iter().any(|word| self.is(index, word)) + } + + /// Whether token `index` follows `.`, `:`, `@` or `$`, where a keyword is a name or path key. + fn name_position(&self, index: usize) -> bool { + index > 0 && matches!(self.lexemes[index - 1].raw.as_str(), "." | ":" | "@" | "$") + } + + fn upper(&self, index: usize) -> String { + self.lexemes[index].raw.to_ascii_uppercase() + } + + fn is_word(&self, index: usize) -> bool { + is_unquoted_word(&self.lexemes[index].raw) + } + + fn operand_ends(&self, index: usize) -> bool { + self.identifiers[index] || ends_operand(&self.lexemes[index]) + } + + fn mark_stages(&mut self) { + if self.dialect != Dialect::Snowflake { + return; + } + let mut index = 0; + while index < self.len() { + let lexeme = &self.lexemes[index]; + let starts = lexeme.raw.starts_with('@') + && matches!(lexeme.kind, TokenType::DAt | TokenType::Var); + if !starts { + index += 1; + continue; + } + self.in_stage[index] = true; + while index + 1 < self.len() + && self.lexemes[index].end == self.lexemes[index + 1].start + && !matches!( + self.lexemes[index + 1].kind, + TokenType::LParen | TokenType::RParen | TokenType::Comma | TokenType::Semicolon + ) + { + index += 1; + self.in_stage[index] = true; + } + index += 1; + } + } + + /// The spacing a one-line rendering puts before token `index`. + fn space(&self, index: usize) -> &'static str { + let (previous, next) = (&self.lexemes[index - 1], &self.lexemes[index]); + if self.in_stage[index] && self.in_stage[index - 1] { + return ""; + } + if self.in_stage[index - 1] && next.kind == TokenType::LParen { + return " "; + } + if matches!( + next.kind, + TokenType::Comma + | TokenType::RParen + | TokenType::RBracket + | TokenType::RBrace + | TokenType::Dot + | TokenType::Semicolon + | TokenType::DColon + | TokenType::Exclamation + | TokenType::Colon + ) { + return ""; + } + if matches!( + previous.kind, + TokenType::LParen + | TokenType::LBracket + | TokenType::LBrace + | TokenType::Dot + | TokenType::DColon + | TokenType::Exclamation + | TokenType::Dollar + ) { + return ""; + } + if previous.kind == TokenType::Colon { + return if self.path_colon(index - 1) { "" } else { " " }; + } + if matches!(previous.kind, TokenType::Dash | TokenType::Plus) + && index + .checked_sub(2) + .is_none_or(|before| !self.operand_ends(before)) + { + return ""; + } + if next.kind == TokenType::LParen { + return if self.spaced_before_parenthesis(index - 1) || !self.calls(index - 1) { + " " + } else { + "" + }; + } + if next.kind == TokenType::LBracket && self.operand_ends(index - 1) { + return ""; + } + " " + } + + /// Whether a `(` right after token `index` opens its argument list. + fn calls(&self, index: usize) -> bool { + self.is_word(index) + || matches!( + self.lexemes[index].kind, + TokenType::QuotedIdentifier | TokenType::RParen | TokenType::RBracket + ) + } + + fn spaced_before_parenthesis(&self, index: usize) -> bool { + if self.identifiers[index] || !self.is_word(index) { + return false; + } + let upper = self.upper(index); + let after_star = index > 0 + && matches!( + self.lexemes[index - 1].kind, + TokenType::Star | TokenType::RParen + ); + let after_sample = + index > 0 && (self.is(index - 1, "SAMPLE") || self.is(index - 1, "TABLESAMPLE")); + SPACED_BEFORE_PARENTHESIS.contains(&upper.as_str()) + || (matches!(upper.as_str(), "EXCLUDE" | "REPLACE" | "RENAME" | "ILIKE") && after_star) + || (upper == SAMPLE_ROW_KEYWORD && after_sample) + } + + /// Whether the `:` at `index` is a Snowflake path accessor such as `v:field`. + fn path_colon(&self, index: usize) -> bool { + self.dialect == Dialect::Snowflake + && index > 0 + && self.lexemes[index].kind == TokenType::Colon + && !matches!( + self.lexemes[index - 1].kind, + TokenType::String | TokenType::DollarString + ) + && self.lexemes.get(index + 1).is_some_and(|next| { + is_unquoted_word(&next.raw) + || matches!(next.kind, TokenType::QuotedIdentifier | TokenType::String) + }) + } + + /// Give every token the width it adds to a one-line rendering of protected SQL. + fn measure(&mut self) { + let mut cast_types: HashMap = HashMap::new(); + for index in 0..self.len() { + if !(self.is(index, "CAST") || self.is(index, "TRY_CAST")) + || self.kind(index + 1) != Some(TokenType::LParen) + { + continue; + } + let Some(closer) = self.partners[index + 1] else { + continue; + }; + if let Some(alias) = self + .tops(index + 2, closer - 1) + .into_iter() + .find(|&position| self.is(position, "AS")) + && alias + 1 < closer + { + cast_types.insert(alias + 1, closer - 1); + } + } + let mut sequence = 0_usize; + let mut index = 0; + while index < self.len() { + let lexeme = &self.lexemes[index]; + let sentinel = sentinel_width(sequence); + let (end, width) = if let Some(&end) = cast_types.get(&index) { + (end, sentinel) + } else if lexeme.kind == TokenType::DColon && index + 1 < self.len() { + ( + self.postfix_type_end(index + 1), + POSTFIX_CAST_WIDTH + sentinel, + ) + } else if self.path_colon(index) { + (index + 1, 1 + sentinel) + } else if TYPED_LITERALS.contains(&lexeme.raw.to_ascii_uppercase().as_str()) + && matches!( + self.kind(index + 1), + Some(TokenType::String | TokenType::DollarString) + ) + { + (index + 1, sentinel) + } else if (self.is(index, "MOD") || self.is(index, "POSITION")) + && self.kind(index + 1) == Some(TokenType::LParen) + { + (index, sentinel) + } else if matches!(lexeme.kind, TokenType::String | TokenType::DollarString) { + (index, sentinel + 2) + } else { + self.widths[index] = lexeme.raw.chars().count(); + index += 1; + continue; + }; + self.widths[index] = width; + for position in index..=end { + self.protected[position] = true; + self.covered[position] = position > index; + } + sequence += 1; + index = end + 1; + } + } + + fn postfix_type_end(&self, start: usize) -> usize { + let mut end = start; + while end + 1 < self.len() && TYPE_CONTINUATIONS.contains(&self.upper(end + 1).as_str()) { + end += 1; + } + if self.kind(end + 1) == Some(TokenType::LParen) + && let Some(closer) = self.partners[end + 1] + { + return closer; + } + end + } + + /// The width of tokens `first..=last` rendered on one line. + fn compact(&self, first: usize, last: usize) -> usize { + (first..=last) + .filter(|&index| !self.covered[index] && !self.dropped[index]) + .map(|index| { + self.widths[index] + + if index > first { + self.base[index].len() + } else { + 0 + } + }) + .sum() + } + + /// The tokens of `first..=last` outside its brackets, brackets included. + fn tops(&self, first: usize, last: usize) -> Vec { + let mut tops: Vec = Vec::new(); + let mut index = first; + while index <= last && index < self.len() { + tops.push(index); + match self.partners[index] { + Some(partner) if partner > index && partner <= last => { + tops.push(partner); + index = partner + 1; + } + _ => index += 1, + } + } + tops + } + + /// The non-empty ranges of `first..=last` between its top-level commas. + fn split_commas(&self, first: usize, last: usize) -> Vec<(usize, usize)> { + let mut items: Vec<(usize, usize)> = Vec::new(); + if first > last { + return items; + } + let mut start = first; + for index in self.tops(first, last) { + if self.lexemes[index].kind == TokenType::Comma { + if start < index { + items.push((start, index - 1)); + } + start = index + 1; + } + } + if start <= last { + items.push((start, last)); + } + items + } + + fn starts_query(&self, index: usize) -> bool { + self.is(index, "SELECT") || self.is(index, "WITH") + } + + fn statements(&mut self) { + let mut first = 0; + for index in 0..self.len() { + if self.lexemes[index].kind != TokenType::Semicolon || self.enclosing[index].is_some() { + continue; + } + if first < index { + self.statement(first, index - 1); + } + self.gaps[index] = String::new(); + first = index + 1; + } + if first < self.len() { + self.statement(first, self.len() - 1); + } + } + + fn statement(&mut self, first: usize, last: usize) { + self.gaps[first] = if first == 0 { + String::new() + } else { + "\n".to_string() + }; + self.query(first, last, 0); + } + + fn query(&mut self, first: usize, last: usize, indent: usize) { + if first > last { + return; + } + let mut start = first; + if self.is(first, "WITH") { + match self.common_table_expressions(first, last, indent) { + Some(main) if main <= last => { + self.gaps[main] = newline(indent); + start = main; + } + Some(_) => return, + None => { + self.expr(first + 1, last, indent); + return; + } + } + } + let operators: Vec = self + .tops(start, last) + .into_iter() + .filter(|&index| self.is_any(index, &SET_OPERATORS)) + .collect(); + let mut operand = start; + for operator in operators { + if operator > operand { + self.operand(operand, operator - 1, indent); + } + self.gaps[operator] = newline(indent); + let mut next = operator + 1; + while next <= last && self.is_any(next, &SET_OPERATOR_MODIFIERS) { + next += 1; + } + if next <= last { + self.gaps[next] = newline(indent); + } + operand = next; + } + self.operand(operand, last, indent); + } + + fn operand(&mut self, first: usize, last: usize, indent: usize) { + if first > last { + return; + } + if self.is(first, "SELECT") { + self.select(first, last, indent); + } else if self.is(first, "WITH") { + self.query(first, last, indent); + } else { + self.expr(first, last, indent); + } + } + + /// Lay out the CTEs of the WITH at `with`; return the token after the last CTE. + fn common_table_expressions( + &mut self, + with: usize, + last: usize, + indent: usize, + ) -> Option { + let mut item = with + 1; + if self.is(item, "RECURSIVE") { + item += 1; + } + loop { + if item > last { + return None; + } + let mut index = item + 1; + if self.kind(index) == Some(TokenType::LParen) + && let Some(closer) = self.partners[index] + && self.is(closer + 1, "AS") + { + index = closer + 1; + } + if self.is(index, "AS") { + index += 1; + if self.is(index, "NOT") { + index += 1; + } + if self.is(index, "MATERIALIZED") { + index += 1; + } + } + if self.kind(index) != Some(TokenType::LParen) { + return None; + } + let closer = self.partners[index].filter(|&closer| closer <= last)?; + self.gaps[index] = " ".to_string(); + self.subquery(index, closer, indent); + if closer < last && self.kind(closer + 1) == Some(TokenType::Comma) { + item = closer + 2; + continue; + } + return Some(closer + 1); + } + } + + fn subquery(&mut self, opener: usize, closer: usize, indent: usize) { + if closer <= opener + 1 { + return; + } + self.gaps[opener + 1] = newline(indent + 1); + self.query(opener + 1, closer - 1, indent + 1); + self.gaps[closer] = newline(indent); + } + + /// Whether a clause starts at `index`, not a keyword alias the parser records without a span. + fn starts_clause(&self, index: usize) -> bool { + let aliased = matches!( + self.kind(index + 1), + Some(TokenType::Comma | TokenType::RParen) + ) || self.is_any(index + 1, &ALIAS_FOLLOWERS); + (!aliased && self.is_any(index, &CLAUSE_KEYWORDS)) + || ((self.is(index, "GROUP") || self.is(index, "ORDER")) && self.is(index + 1, "BY")) + } + + fn select(&mut self, select: usize, last: usize, indent: usize) { + let clauses: Vec = self + .tops(select + 1, last) + .into_iter() + .filter(|&index| self.starts_clause(index)) + .collect(); + let projection_end = clauses.first().map_or(last, |&clause| clause - 1); + let mut item = select + 1; + loop { + if item > projection_end { + break; + } + if self.is(item, "DISTINCT") || self.is(item, "ALL") { + item += 1; + if self.is(item - 1, "DISTINCT") + && self.is(item, "ON") + && let Some(closer) = self.partners.get(item + 1).copied().flatten() + { + item = closer + 1; + } + continue; + } + if self.is(item, "TOP") { + item += 1; + item = match self.partners.get(item).copied().flatten() { + Some(closer) if closer > item => closer + 1, + _ => item + 1, + }; + continue; + } + break; + } + self.selects += 1; + for (first, end) in self.split_commas(item, projection_end) { + self.gaps[first] = newline(indent + 1); + if !self.aliases[end] && !self.plain_column(first, end) { + self.verbatim.push((first, end)); + } + self.expr(first, end, indent + 1); + } + for (position, &clause) in clauses.iter().enumerate() { + let end = clauses.get(position + 1).map_or(last, |&next| next - 1); + self.clause(clause, end, indent); + } + } + + /// Whether tokens `first..=last` are a column reference such as `t.a` or `*`, which names itself. + fn plain_column(&self, first: usize, last: usize) -> bool { + (first..=last).all(|index| { + self.identifiers[index] + || matches!(self.lexemes[index].kind, TokenType::Dot | TokenType::Star) + }) + } + + fn clause(&mut self, clause: usize, end: usize, indent: usize) { + self.gaps[clause] = newline(indent); + match self.upper(clause).as_str() { + "FROM" => self.from(clause + 1, end, indent), + "WHERE" | "HAVING" | "QUALIFY" => { + if clause < end { + self.gaps[clause + 1] = newline(indent + 1); + self.expr(clause + 1, end, indent + 1); + } + } + "GROUP" | "ORDER" => { + let body = clause + 2; + if body > end || (self.is(body, "ALL") && body == end) { + return; + } + self.items(body, end, indent); + } + "WINDOW" => self.items(clause + 1, end, indent), + _ => self.expr(clause + 1, end, indent), + } + } + + fn items(&mut self, first: usize, last: usize, indent: usize) { + for (start, end) in self.split_commas(first, last) { + self.gaps[start] = newline(indent + 1); + self.expr(start, end, indent + 1); + } + } + + fn is_join_word(&self, index: usize) -> bool { + self.is(index, "JOIN") || self.is_any(index, &JOIN_PREFIXES) + } + + fn starts_join(&self, index: usize, first: usize) -> bool { + if !self.is_join_word(index) || (index > first && self.is_join_word(index - 1)) { + return false; + } + let mut position = index; + while position < self.len() && self.is_join_word(position) && !self.is(position, "JOIN") { + position += 1; + } + self.is(position, "JOIN") + } + + fn from(&mut self, first: usize, last: usize, indent: usize) { + if first > last { + return; + } + let mut segments: Vec<(usize, usize, bool)> = Vec::new(); + let (mut start, mut joined) = (first, false); + for index in self.tops(first, last) { + if self.lexemes[index].kind == TokenType::Comma { + if start < index { + segments.push((start, index - 1, joined)); + } + (start, joined) = (index + 1, false); + } else if index > start && self.starts_join(index, first) { + segments.push((start, index - 1, joined)); + self.gaps[index] = newline(indent); + (start, joined) = (index, true); + } + } + if start <= last { + segments.push((start, last, joined)); + } + for (start, end, joined) in segments { + if joined { + self.join(start, end, indent); + } else { + self.source(start, end, indent); + } + } + } + + fn join(&mut self, first: usize, last: usize, indent: usize) { + let mut join = first; + while join < last && !self.is(join, "JOIN") { + join += 1; + } + let condition = self + .tops(join + 1, last) + .into_iter() + .find(|&index| self.is(index, "ON") || self.is(index, "USING")); + self.source(join + 1, condition.map_or(last, |at| at - 1), indent); + if let Some(condition) = condition { + self.gaps[condition] = newline(indent + 1); + self.expr(condition + 1, last, indent + 1); + } + } + + fn source(&mut self, first: usize, last: usize, indent: usize) { + if first > last { + return; + } + let opener = if self.is(first, "LATERAL") { + first + 1 + } else { + first + }; + if self.kind(opener) == Some(TokenType::LParen) + && let Some(closer) = self.partners[opener].filter(|&closer| closer <= last) + { + if self.starts_query(opener + 1) { + self.subquery(opener, closer, indent); + } else { + self.from(opener + 1, closer - 1, indent); + } + self.expr(closer + 1, last, indent); + return; + } + self.expr(first, last, indent); + } + + fn expr(&mut self, first: usize, last: usize, indent: usize) { + let mut between = false; + let mut index = first; + while index <= last && index < self.len() { + if self.is(index, "SELECT") { + self.query(index, last, indent); + return; + } + match self.lexemes[index].kind { + TokenType::LParen | TokenType::LBracket | TokenType::LBrace => { + if let Some(closer) = self.partners[index].filter(|&closer| closer <= last) { + self.group(index, closer, indent); + index = closer + 1; + continue; + } + } + _ => {} + } + if self.is(index, "CASE") + && let Some(end) = self.case_end(index, last) + { + self.case(index, end, indent); + index = end + 1; + continue; + } + if self.is(index, "BETWEEN") { + between = true; + } else if self.is(index, "AND") && between { + between = false; + } else if (self.is(index, "AND") || self.is(index, "OR")) + && self.dialect == Dialect::Snowflake + && !(index > 0 && self.is(index - 1, "CREATE")) + { + self.gaps[index] = newline(indent); + } + index += 1; + } + } + + fn group(&mut self, opener: usize, closer: usize, indent: usize) { + match self.lexemes[opener].kind { + TokenType::LBracket => { + if self.base[opener].is_empty() && opener > 0 && self.operand_ends(opener - 1) { + self.expr(opener + 1, closer - 1, indent); + } else { + let items = self.split_commas(opener + 1, closer - 1); + let split = !items.is_empty() && self.items_width(&items) > GUIDE_WIDTH; + self.list(closer, &items, split, indent); + } + } + TokenType::LBrace => { + if opener + 1 < closer && self.lexemes[opener + 1].raw.eq_ignore_ascii_case("fn") { + self.dropped[opener] = true; + self.dropped[opener + 1] = true; + self.dropped[closer] = true; + self.expr(opener + 2, closer - 1, indent); + } else { + self.expr(opener + 1, closer - 1, indent); + } + } + _ => self.parenthesis(opener, closer, indent), + } + } + + fn parenthesis(&mut self, opener: usize, closer: usize, indent: usize) { + let previous = opener.checked_sub(1); + if self.starts_query(opener + 1) { + let pivot_list = previous.is_some_and(|at| self.is(at, "IN")) + && self.enclosing[opener].is_some_and(|at| { + at > 0 && (self.is(at - 1, "PIVOT") || self.is(at - 1, "UNPIVOT")) + }); + if pivot_list { + self.query(opener + 1, closer - 1, indent); + } else { + self.subquery(opener, closer, indent); + } + return; + } + let Some(previous) = previous else { + self.tuple(opener, closer, indent); + return; + }; + let pivot_values = self.is(previous, "IN") + && self.enclosing[opener].is_some_and(|at| at > 0 && self.is(at - 1, "PIVOT")); + if (self.is(previous, "PATTERN") || pivot_values) && opener + 1 < closer { + self.verbatim.push((opener + 1, closer - 1)); + self.expr(opener + 1, closer - 1, indent); + } else if self.is(previous, "PIVOT") || self.is(previous, "UNPIVOT") { + self.expr(opener + 1, closer - 1, indent); + } else if self.is_any(previous, &GROUPING_CALLS) + || (self.base[opener].is_empty() && self.calls(previous) && !self.in_stage[previous]) + { + self.call(previous, opener, closer, indent); + } else if self.is_word(previous) || self.in_stage[previous] { + self.expr(opener + 1, closer - 1, indent); + } else { + self.tuple(opener, closer, indent); + } + } + + fn tuple(&mut self, opener: usize, closer: usize, indent: usize) { + let items = self.split_commas(opener + 1, closer - 1); + let split = items.len() > 1 && self.items_width(&items) > GUIDE_WIDTH; + self.list(closer, &items, split, indent); + } + + fn call(&mut self, name: usize, opener: usize, closer: usize, indent: usize) { + let upper = if self.is_word(name) { + self.upper(name) + } else { + String::new() + }; + if upper == SEMANTIC_VIEW_CALL { + self.semantic_view(opener, closer, indent); + return; + } + let mut first = opener + 1; + if self.is(first, "DISTINCT") { + first += 1; + } + let items = self.split_commas(first, closer - 1); + let grouping = GROUPING_CALLS.contains(&upper.as_str()); + let inline = self.protected[opener] || self.inline_calls().contains(&upper.as_str()); + let split = + !items.is_empty() && (grouping || (!inline && self.items_width(&items) > GUIDE_WIDTH)); + self.list(closer, &items, split, indent); + } + + fn inline_calls(&self) -> &'static [&'static str] { + match self.dialect { + Dialect::Snowflake => &SNOWFLAKE_INLINE_CALLS, + Dialect::DuckDB => &DUCKDB_INLINE_CALLS, + Dialect::PostgreSQL | Dialect::SQLite => &POSTGRES_INLINE_CALLS, + } + } + + fn items_width(&self, items: &[(usize, usize)]) -> usize { + items + .iter() + .map(|&(first, last)| self.compact(first, last)) + .sum() + } + + /// Lay out the items of a bracket group closed at `closer`, one per line when `split`. + fn list(&mut self, closer: usize, items: &[(usize, usize)], split: bool, indent: usize) { + if !split { + for &(first, last) in items { + self.expr(first, last, indent); + } + return; + } + for &(first, last) in items { + self.gaps[first] = newline(indent + 1); + self.expr(first, last, indent + 1); + } + self.gaps[closer] = newline(indent); + } + + fn semantic_view(&mut self, opener: usize, closer: usize, indent: usize) { + if closer <= opener + 1 { + return; + } + let clauses: Vec = self + .tops(opener + 1, closer - 1) + .into_iter() + .filter(|&index| self.is_any(index, &SEMANTIC_VIEW_CLAUSES)) + .collect(); + let mut start = opener + 1; + for clause in clauses.iter().copied().chain([closer]) { + if start < clause { + self.gaps[start] = newline(indent + 1); + self.expr(start, clause - 1, indent + 1); + } + start = clause; + } + self.gaps[closer] = newline(indent); + } + + fn case_end(&self, case: usize, last: usize) -> Option { + let mut nested = 0_usize; + for index in self.tops(case + 1, last) { + if self.is(index, "CASE") { + nested += 1; + } else if self.is(index, "END") { + if nested == 0 { + return Some(index); + } + nested -= 1; + } + } + None + } + + fn case(&mut self, case: usize, end: usize, indent: usize) { + let mut nested = 0_usize; + let mut branches: Vec = Vec::new(); + for index in self.tops(case + 1, end - 1) { + if self.is(index, "CASE") { + nested += 1; + } else if self.is(index, "END") { + nested = nested.saturating_sub(1); + } else if nested == 0 && self.is_any(index, &CASE_BRANCHES) { + branches.push(index); + } + } + let width = self.compact(case, end).saturating_sub(branches.len() + 1); + let multiline = width > GUIDE_WIDTH; + let inner = if multiline { indent + 1 } else { indent }; + let first_branch = branches.first().copied().unwrap_or(end); + self.expr(case + 1, first_branch - 1, indent); + for (position, &branch) in branches.iter().enumerate() { + if multiline { + self.gaps[branch] = newline(indent + 1); + } + let next = branches.get(position + 1).copied().unwrap_or(end); + self.expr(branch + 1, next - 1, inner); + } + if multiline { + self.gaps[end] = newline(indent); + } + } + + /// The guide's spelling of token `index`: keywords in upper case, names as written. + fn text(&self, index: usize) -> String { + let lexeme = &self.lexemes[index]; + if self.protected[index] + || self.in_stage[index] + || self.identifiers[index] + || self.name_position(index) + || self.field_name(index) + || !is_keyword(lexeme) + { + lexeme.raw.clone() + } else { + lexeme.raw.to_ascii_uppercase() + } + } + + /// Whether token `index` names a field of a struct type such as `STRUCT(type VARCHAR)` or `STRUCT`; the tree keeps these names without spans, and a schema keeps their case. + fn field_name(&self, index: usize) -> bool { + let starts_field = index > 0 + && matches!( + self.lexemes[index - 1].kind, + TokenType::Comma | TokenType::LParen | TokenType::Lt + ); + let typed = self + .lexemes + .get(index + 1) + .is_some_and(|next| is_unquoted_word(&next.raw) || next.kind == TokenType::Colon); + if !starts_field || !typed { + return false; + } + let mut angles: usize = 0; + let mut at = index; + while at > 0 { + at -= 1; + match self.lexemes[at].kind { + TokenType::Gt => angles += 1, + TokenType::GtGt => angles += 2, + TokenType::RParen | TokenType::RBracket => { + if let Some(opener) = self.partners[at].filter(|&opener| opener < at) { + at = opener; + } + } + TokenType::Lt if angles > 0 => angles -= 1, + TokenType::Lt | TokenType::LParen | TokenType::LBracket => { + return angles == 0 + && self.lexemes[at].kind != TokenType::LBracket + && at > 0 + && STRUCT_TYPE_WORDS + .iter() + .any(|word| self.lexemes[at - 1].raw.eq_ignore_ascii_case(word)); + } + _ => {} + } + } + false + } + + fn render(&self) -> GuideLayout { + let mut text = String::new(); + let mut tokens: Vec = Vec::with_capacity(self.len()); + let mut pending: Option<&str> = None; + for index in 0..self.len() { + if self.dropped[index] { + pending.get_or_insert(&self.gaps[index]); + continue; + } + if !tokens.is_empty() { + let gap = match pending { + Some(gap) if gap.contains('\n') => gap, + _ => &self.gaps[index], + }; + text.push_str(gap); + } + pending = None; + let start = text.len(); + let raw = self.text(index); + text.push_str(&raw); + tokens.push(Lexeme { + start, + end: text.len(), + kind: self.lexemes[index].kind, + raw, + }); + } + let mut ranges = self.verbatim.clone(); + ranges.sort_unstable(); + let mut verbatim: Vec<(usize, usize)> = Vec::with_capacity(ranges.len()); + for range in ranges { + if verbatim.last().is_none_or(|&(_, end)| range.0 > end) { + verbatim.push(range); + } + } + GuideLayout { + text, + tokens, + verbatim, + selects: self.selects, + } + } +} diff --git a/crates/sqlquack-format/src/format/_helpers/identifiers.rs b/crates/sqlquack-format/src/format/_helpers/identifiers.rs new file mode 100644 index 000000000..deeddd936 --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/identifiers.rs @@ -0,0 +1,256 @@ +//! What the parse tree says about the tokens: which name identifiers or aliases, which are binary operators. + +use serde_json::Value; +use sqlquack_syntax::Expression; + +use crate::format::_helpers::lexemes::Lexeme; +use crate::format::constants::BINARY_OPERATOR_NODES; + +/// The fields of a serialised identifier node. +const IDENTIFIER_FIELDS: [&str; 4] = ["name", "quoted", "trailing_comments", "span"]; +/// The field of a serialised node that names its alias, and of a select node. +const ALIAS_FIELD: &str = "alias"; +const ALIASED_FIELD: &str = "this"; +const SELECT_FIELD: &str = "select"; +const PROJECTION_FIELD: &str = "expressions"; +const SPAN_FIELD: &str = "span"; +const LEFT_FIELD: &str = "left"; +const RIGHT_FIELD: &str = "right"; + +/// Per-token facts from the parse tree, and how many select lists it has. +pub(crate) struct ParsedNames { + /// Tokens inside the source range of an identifier. + pub(crate) identifiers: Vec, + /// Tokens inside the source range of a name given with an alias, such as `b` in `a AS b`. + pub(crate) aliases: Vec, + /// Tokens the parse tree reads as the operator of a binary comparison, arithmetic or concatenation. + pub(crate) binary_operators: Vec, + /// The number of select lists in the parse tree. + pub(crate) selects: usize, +} + +/// A binary operator node: its spellings, and its range and where its operands end and start when recorded. +struct Binary { + spellings: &'static [&'static str], + span: Option<(usize, usize)>, + left_end: Option, + right_start: Option, +} + +/// The ranges the parse tree records: every identifier and every alias name, in characters. +#[derive(Default)] +struct Ranges { + identifiers: Vec<(usize, usize)>, + aliases: Vec<(usize, usize)>, + /// Every recorded node range. + spans: Vec<(usize, usize)>, + binaries: Vec, + selects: usize, +} + +/// Mark the tokens of `lexemes` the parse trees of `statements` name identifiers or aliases. +pub(crate) fn parsed_names( + sql: &str, + statements: &[Expression], + lexemes: &[Lexeme], +) -> Result { + let bytes: Vec = sql + .char_indices() + .map(|(byte, _)| byte) + .chain([sql.len()]) + .collect(); + let byte = |offset: usize| bytes.get(offset).copied().unwrap_or(sql.len()); + let trees: Vec = statements + .iter() + .map(serde_json::to_value) + .collect::>() + .map_err(|error| error.to_string())?; + let ranges = trees + .iter() + .map(collect) + .fold(Ranges::default(), Ranges::merged); + let to_bytes = |ranges: &[(usize, usize)]| -> Vec<(usize, usize)> { + let mut converted: Vec<(usize, usize)> = ranges + .iter() + .map(|&(start, end)| (byte(start), byte(end))) + .collect(); + converted.sort_unstable(); + converted + }; + let (identifiers, aliases) = (to_bytes(&ranges.identifiers), to_bytes(&ranges.aliases)); + Ok(ParsedNames { + identifiers: lexemes + .iter() + .map(|lexeme| covered(&identifiers, lexeme)) + .collect(), + aliases: lexemes + .iter() + .map(|lexeme| covered(&aliases, lexeme)) + .collect(), + binary_operators: binary_operators(&ranges, lexemes, &byte), + selects: ranges.selects, + }) +} + +/// For every token, whether a binary operator node of the parse tree has it as its operator. +fn binary_operators( + ranges: &Ranges, + lexemes: &[Lexeme], + byte: &dyn Fn(usize) -> usize, +) -> Vec { + let mut operators: Vec = vec![false; lexemes.len()]; + let spans: Vec<(usize, usize)> = ranges + .spans + .iter() + .map(|&(start, end)| (byte(start), byte(end))) + .collect(); + for binary in &ranges.binaries { + let (start, end) = binary + .span + .map_or((0, usize::MAX), |(start, end)| (byte(start), byte(end))); + let inside: Vec = (0..lexemes.len()) + .filter(|&index| lexemes[index].start >= start && lexemes[index].end <= end) + .collect(); + let found = match (binary.left_end.map(byte), binary.right_start.map(byte)) { + (Some(left_end), _) => inside + .iter() + .copied() + .find(|&index| lexemes[index].start >= left_end), + (None, Some(right_start)) => inside + .iter() + .copied() + .rfind(|&index| lexemes[index].end <= right_start), + (None, None) if binary.span.is_some() => { + let candidates: Vec = inside + .iter() + .copied() + .filter(|&index| { + binary.spellings.contains(&lexemes[index].raw.as_str()) + && !in_inner_node(&spans, (start, end), &lexemes[index]) + }) + .collect(); + match candidates.as_slice() { + [only] => Some(*only), + _ => None, + } + } + (None, None) => None, + }; + if let Some(index) = + found.filter(|&index| binary.spellings.contains(&lexemes[index].raw.as_str())) + { + operators[index] = true; + } + } + operators +} + +/// Whether one of the sorted byte `ranges` covers `lexeme`. +fn covered(ranges: &[(usize, usize)], lexeme: &Lexeme) -> bool { + let before = ranges.partition_point(|&(start, _)| start <= lexeme.start); + ranges[..before].iter().any(|&(_, end)| end >= lexeme.end) +} + +/// The identifier and alias ranges and the select lists of `value` and every node under it, walked without recursion so a deep tree cannot exhaust the stack. +fn collect(value: &Value) -> Ranges { + let mut ranges = Ranges::default(); + let mut pending: Vec<&Value> = vec![value]; + while let Some(node) = pending.pop() { + match node { + Value::Object(fields) => pending.extend(fields.values()), + Value::Array(items) => pending.extend(items), + _ => {} + } + ranges = ranges.merged(own_ranges(node)); + } + ranges +} + +/// The identifier and alias ranges and the select list `value` itself records, without its children. +fn own_ranges(value: &Value) -> Ranges { + Ranges { + identifiers: identifier_range(value).into_iter().collect(), + aliases: value + .get(ALIAS_FIELD) + .filter(|node| node.get(ALIASED_FIELD).is_some()) + .and_then(|node| node.get(ALIAS_FIELD)) + .and_then(identifier_range) + .into_iter() + .collect(), + spans: value + .get(SPAN_FIELD) + .and_then(|span| Some((span_offset(span, "start")?, span_offset(span, "end")?))) + .into_iter() + .collect(), + binaries: binary(value).into_iter().collect(), + selects: usize::from( + value + .get(SELECT_FIELD) + .is_some_and(|node| node.get(PROJECTION_FIELD).is_some()), + ), + } +} + +impl Ranges { + /// Both sets of ranges and select lists together. + fn merged(mut self, other: Ranges) -> Ranges { + self.identifiers.extend(other.identifiers); + self.aliases.extend(other.aliases); + self.spans.extend(other.spans); + self.binaries.extend(other.binaries); + self.selects += other.selects; + self + } +} + +/// Whether `lexeme` lies in a recorded node range strictly inside the range `outer`. +fn in_inner_node(spans: &[(usize, usize)], outer: (usize, usize), lexeme: &Lexeme) -> bool { + spans.iter().any(|&(start, end)| { + (start, end) != outer + && start >= outer.0 + && end <= outer.1 + && start <= lexeme.start + && lexeme.end <= end + }) +} + +/// The binary operator node `value` is, when it is one with a recorded span. +fn binary(value: &Value) -> Option { + let fields = value.as_object().filter(|fields| fields.len() == 1)?; + let (kind, node) = fields.iter().next()?; + let (_, spellings) = BINARY_OPERATOR_NODES + .iter() + .find(|(name, _)| name == kind)?; + let operand_span = |field: &str| -> Option<&Value> { + node.get(field)? + .as_object()? + .values() + .next()? + .get(SPAN_FIELD) + }; + Some(Binary { + spellings, + span: node + .get(SPAN_FIELD) + .and_then(|span| Some((span_offset(span, "start")?, span_offset(span, "end")?))), + left_end: operand_span(LEFT_FIELD).and_then(|span| span_offset(span, "end")), + right_start: operand_span(RIGHT_FIELD).and_then(|span| span_offset(span, "start")), + }) +} + +/// The character range of `value` when it is an identifier node with a recorded span. +fn identifier_range(value: &Value) -> Option<(usize, usize)> { + let fields = value.as_object()?; + let identifier = fields.get("name").is_some_and(Value::is_string) + && fields.get("quoted").is_some_and(Value::is_boolean) + && fields + .keys() + .all(|key| IDENTIFIER_FIELDS.contains(&key.as_str())); + let span = fields.get("span").filter(|_| identifier)?; + Some((span_offset(span, "start")?, span_offset(span, "end")?)) +} + +/// One character offset of a serialised span. +fn span_offset(span: &Value, key: &str) -> Option { + span.get(key)?.as_u64().map(|offset| offset as usize) +} diff --git a/crates/sqlquack-format/src/format/_helpers/lexemes.rs b/crates/sqlquack-format/src/format/_helpers/lexemes.rs new file mode 100644 index 000000000..f0b0a810a --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/lexemes.rs @@ -0,0 +1,132 @@ +//! The significant tokens and comments of SQL, read from the lossless tokenizer. + +use sqlquack_syntax::tokens::models::TokenType; +use sqlquack_syntax::{Dialect, TokenKind, tokenize}; + +use crate::format::constants::TOKENIZE_FAILURE; + +/// One significant token with its byte range and source text. +#[derive(Debug, Clone)] +pub(crate) struct Lexeme { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) kind: TokenType, + pub(crate) raw: String, +} + +/// A comment between significant tokens. +#[derive(Debug, Clone)] +pub(crate) struct Comment { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) text: String, + /// A comment that runs to the end of its line. + pub(crate) line: bool, + /// Only whitespace precedes the comment on its line. + pub(crate) leading: bool, +} + +/// The significant tokens and the comments of `sql`. +pub(crate) fn lex(sql: &str, dialect: Dialect) -> Result<(Vec, Vec), String> { + let tokens = tokenize(sql, dialect).map_err(|error| format!("{TOKENIZE_FAILURE}: {error}"))?; + let mut lexemes: Vec = Vec::with_capacity(tokens.len()); + let mut comments: Vec = Vec::new(); + for token in &tokens { + let text = token.text(sql); + match token.kind { + TokenKind::Significant(kind) => lexemes.push(Lexeme { + start: token.start, + end: token.end, + kind, + raw: text.to_string(), + }), + TokenKind::LineComment | TokenKind::BlockComment => { + let line_start = sql[..token.start].rfind('\n').map_or(0, |at| at + 1); + comments.push(Comment { + start: token.start, + end: token.end, + text: text.trim_end_matches(['\r', '\n']).to_string(), + line: token.kind == TokenKind::LineComment, + leading: sql[line_start..token.start].trim().is_empty(), + }); + } + TokenKind::Whitespace => {} + } + } + Ok((lexemes, comments)) +} + +/// Whether `text` is an unquoted ASCII word whose letter case carries no meaning. +pub(crate) fn is_unquoted_word(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '$')) +} + +/// Whether the tokenizer types this unquoted word as a keyword rather than a name. +pub(crate) fn is_keyword(lexeme: &Lexeme) -> bool { + is_unquoted_word(&lexeme.raw) + && !matches!( + lexeme.kind, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ) +} + +/// Whether the token can end an operand, so a `-` or `[` after it is binary or a subscript. +pub(crate) fn ends_operand(lexeme: &Lexeme) -> bool { + matches!( + lexeme.kind, + TokenType::Var + | TokenType::Identifier + | TokenType::QuotedIdentifier + | TokenType::Number + | TokenType::Parameter + | TokenType::RParen + | TokenType::RBracket + | TokenType::RBrace + | TokenType::Null + | TokenType::True + | TokenType::False + | TokenType::End + ) || is_string_literal(lexeme.kind) +} + +/// Whether the token type is a string literal of any quoting. +pub(crate) fn is_string_literal(kind: TokenType) -> bool { + matches!( + kind, + TokenType::String + | TokenType::NationalString + | TokenType::EscapeString + | TokenType::RawString + | TokenType::ByteString + | TokenType::UnicodeString + | TokenType::HexString + | TokenType::BitString + | TokenType::DollarString + | TokenType::TripleSingleQuotedString + | TokenType::TripleDoubleQuotedString + ) +} + +/// Each token's bracket partner, for `(`, `[` and `{`. +pub(crate) fn bracket_partners(lexemes: &[Lexeme]) -> Vec> { + let mut partners: Vec> = vec![None; lexemes.len()]; + let mut open: Vec = Vec::new(); + for (index, lexeme) in lexemes.iter().enumerate() { + match lexeme.kind { + TokenType::LParen | TokenType::LBracket | TokenType::LBrace => open.push(index), + TokenType::RParen | TokenType::RBracket | TokenType::RBrace => { + if let Some(opener) = open.pop() { + partners[opener] = Some(index); + partners[index] = Some(opener); + } + } + _ => {} + } + } + partners +} diff --git a/crates/sqlquack-format/src/format/_helpers/mod.rs b/crates/sqlquack-format/src/format/_helpers/mod.rs new file mode 100644 index 000000000..2b322f23d --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/mod.rs @@ -0,0 +1,9 @@ +//! Helpers of format. + +pub(crate) mod guide; +pub(crate) mod identifiers; +pub(crate) mod lexemes; +pub(crate) mod pipeline; +pub(crate) mod printed_layout; +pub(crate) mod token_layout; +pub(crate) mod touching; diff --git a/crates/sqlquack-format/src/format/_helpers/pipeline.rs b/crates/sqlquack-format/src/format/_helpers/pipeline.rs new file mode 100644 index 000000000..48868093b --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/pipeline.rs @@ -0,0 +1,223 @@ +//! The formatting pipeline: lex, parse, lay out against the guide, then check the output. + +use sqlquack_syntax::tokens::models::TokenType; +use sqlquack_syntax::{CallMode, Dialect, ExpressionWalk, parse_statements_with}; + +use crate::format::_helpers::guide::layout_guide; +use crate::format::_helpers::identifiers::parsed_names; +use crate::format::_helpers::lexemes::{Lexeme, ends_operand, is_unquoted_word, lex}; +use crate::format::_helpers::printed_layout::{PrintedSql, wrap_lines}; +use crate::format::_helpers::token_layout::{ + AuthoredPrint, AuthoredSql, Oracle, print_authored_tokens, verify_printed_tokens, +}; +use crate::format::_helpers::touching::verify_touching_tokens; +use crate::format::constants::{ + DEPTH_FAILURE, MAX_TREE_DEPTH, MAX_TYPE_NESTING, NOT_IDEMPOTENT_FAILURE, PARSE_FAILURE, + SELECT_LIST_FAILURE, TOKEN_PRESERVATION_FAILURE, TYPE_CONSTRUCTOR_WORDS, TYPE_DEPTH_FAILURE, + UNSUPPORTED_DIALECT_FAILURE, UNSUPPORTED_SYNTAX_FAILURE, +}; +use crate::format::models::{FormatOptions, Formatted}; + +/// Format `sql`, or return why it is left as written. +pub(crate) fn format_checked(sql: &str, options: &FormatOptions) -> Result { + let dialect = options.dialect; + if dialect != Dialect::Snowflake { + return Err(UNSUPPORTED_DIALECT_FAILURE.to_string()); + } + let (lexemes, comments) = lex(sql, dialect)?; + if lexemes.is_empty() { + return Ok(Formatted { + sql: sql.to_string(), + changed: false, + }); + } + if dialect == Dialect::Snowflake && has_unsupported_syntax(&lexemes) { + return Err(UNSUPPORTED_SYNTAX_FAILURE.to_string()); + } + if nests_types_deeply(&lexemes) { + return Err(TYPE_DEPTH_FAILURE.to_string()); + } + let statements = parse_statements_with(sql, dialect, &options.parse, CallMode::FaithfulLocated) + .map_err(|error| format!("{PARSE_FAILURE}: {error}"))?; + if statements + .iter() + .any(|statement| statement.tree_depth() > MAX_TREE_DEPTH) + { + return Err(DEPTH_FAILURE.to_string()); + } + let names = parsed_names(sql, &statements, &lexemes)?; + let guide = layout_guide(&lexemes, &names, dialect); + if guide.selects != names.selects { + return Err(SELECT_LIST_FAILURE.to_string()); + } + let oracle = Oracle::new(guide); + let authored = AuthoredSql { + sql, + lexemes: &lexemes, + comments: &comments, + }; + let print = print_layout(&authored, &oracle, options)?; + if print.sql == sql { + return Ok(Formatted { + sql: print.sql, + changed: false, + }); + } + let (printed_lexemes, printed_comments) = lex(&print.sql, dialect)?; + let printed = AuthoredSql { + sql: &print.sql, + lexemes: &printed_lexemes, + comments: &printed_comments, + }; + verify_printed_tokens(&authored, &printed, &print.recases)?; + verify_touching_tokens(&authored, &printed, &names.binary_operators)?; + if print_layout(&printed, &oracle, options)?.sql != print.sql { + return Err(NOT_IDEMPOTENT_FAILURE.to_string()); + } + Ok(Formatted { + sql: print.sql, + changed: true, + }) +} + +/// Print `authored` against the guide, then apply the clause rules and optional line wrapping. +fn print_layout( + authored: &AuthoredSql<'_>, + oracle: &Oracle, + options: &FormatOptions, +) -> Result { + let print = print_authored_tokens(authored, oracle)?; + let Some(mut sql) = PrintedSql::new(&print.sql, &print.tokens, &options.token_widths) else { + return Ok(print); + }; + sql.set_indent(options.indent); + sql.attach_commas_before_comments(); + if let Some(line_width) = options.line_width { + sql.join_fitting_groups(line_width); + } + sql.arrange_clauses(options.line_width.unwrap_or(usize::MAX)); + if let Some(line_width) = options.line_width { + wrap_lines(&mut sql, line_width); + } + sql.lead_operators_of_multiline_operands(); + Ok(AuthoredPrint { + sql: keep_verbatim(authored, sql.render(), &oracle.verbatim, options.dialect)?, + ..print + }) +} + +/// `printed` with the text of each `verbatim` token range put back as `authored` wrote it. +fn keep_verbatim( + authored: &AuthoredSql<'_>, + printed: String, + verbatim: &[(usize, usize)], + dialect: Dialect, +) -> Result { + if verbatim.is_empty() { + return Ok(printed); + } + let (lexemes, _) = lex(&printed, dialect)?; + if lexemes.len() != authored.lexemes.len() { + return Err(TOKEN_PRESERVATION_FAILURE.to_string()); + } + let mut kept = printed; + for &(first, last) in verbatim.iter().rev() { + let text = &authored.sql[authored.lexemes[first].start..authored.lexemes[last].end]; + kept.replace_range(lexemes[first].start..lexemes[last].end, text); + } + Ok(kept) +} + +/// Whether a data type nests deeper than `MAX_TYPE_NESTING`, counting bracket suffixes such as `INT[]` and constructors such as `ARRAY<...`, `ARRAY(...` or `"ARRAY"(...` along one chain, since the parser does not bound either; each open constructor keeps its fields' starting depth, its deepest field, and whether it opened with `(`. +fn nests_types_deeply(lexemes: &[Lexeme]) -> bool { + let mut frames: Vec<(usize, usize, bool)> = Vec::new(); + let mut paren_opens_type: Vec = Vec::new(); + let mut depth: usize = 0; + let mut closed_type = false; + for (index, lexeme) in lexemes.iter().enumerate() { + let after_type_word = index.checked_sub(1).is_some_and(|at| { + let name = lexemes[at].raw.trim_matches(['"', '`']); + TYPE_CONSTRUCTOR_WORDS + .iter() + .any(|word| name.eq_ignore_ascii_case(word)) + }); + let continues = closed_type + || index + .checked_sub(1) + .is_some_and(|at| lexemes[at].kind == TokenType::RBracket); + closed_type = false; + match lexeme.kind { + TokenType::LBracket if continues => depth += 1, + TokenType::LBracket => depth = frames.last().map_or(0, |&(base, _, _)| base) + 1, + TokenType::Lt | TokenType::LParen if after_type_word => { + depth = if frames.is_empty() { 1 } else { depth + 1 }; + let paren = lexeme.kind == TokenType::LParen; + frames.push((depth, depth, paren)); + if paren { + paren_opens_type.push(true); + } + } + TokenType::LParen => paren_opens_type.push(false), + TokenType::RParen => { + if paren_opens_type.pop() == Some(true) { + while let Some((_, deepest, paren)) = frames.pop() { + depth = depth.max(deepest); + if paren { + break; + } + } + closed_type = true; + } + } + TokenType::Gt | TokenType::GtGt => { + let closes = if lexeme.kind == TokenType::GtGt { 2 } else { 1 }; + for _ in 0..closes { + if frames.last().is_some_and(|&(_, _, paren)| !paren) { + if let Some((_, deepest, _)) = frames.pop() { + depth = depth.max(deepest); + } + closed_type = true; + } + } + } + TokenType::Comma => { + if let Some((base, deepest, _)) = frames.last_mut() { + *deepest = (*deepest).max(depth); + depth = *base; + } + } + _ => {} + } + if depth > MAX_TYPE_NESTING || frames.len() > MAX_TYPE_NESTING { + return true; + } + } + false +} + +/// Bind variables such as `:1`, and string stage references with options, are left as written. +fn has_unsupported_syntax(lexemes: &[Lexeme]) -> bool { + (0..lexemes.len()) + .any(|index| is_bind_variable(lexemes, index) || is_staged_string(lexemes, index)) +} + +/// Whether a `:` at `index` starts a bind variable such as `:1` or `:name`. +fn is_bind_variable(lexemes: &[Lexeme], index: usize) -> bool { + let after_operand = index + .checked_sub(1) + .is_some_and(|previous| ends_operand(&lexemes[previous])); + lexemes[index].kind == TokenType::Colon + && !after_operand + && lexemes + .get(index + 1) + .is_some_and(|next| next.kind == TokenType::Number || is_unquoted_word(&next.raw)) +} + +/// Whether a string stage reference such as `'@stage'` at `index` is followed by options. +fn is_staged_string(lexemes: &[Lexeme], index: usize) -> bool { + lexemes[index].kind == TokenType::String + && lexemes[index].raw.starts_with("'@") + && lexemes + .get(index + 1) + .is_some_and(|next| next.kind == TokenType::LParen) +} diff --git a/crates/sqlquack-format/src/format/_helpers/printed_layout.rs b/crates/sqlquack-format/src/format/_helpers/printed_layout.rs new file mode 100644 index 000000000..272d7853b --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/printed_layout.rs @@ -0,0 +1,1066 @@ +//! Printed SQL as tokens and gaps, with clause layout rules and line-width wrapping. + +use std::collections::HashMap; +use std::ops::Range; + +use sqlquack_syntax::tokens::models::TokenType; + +const OPEN_PARENTHESIS: &str = "("; + +/// The indent step of the parse-tree layout every printed statement starts from. +const PARSED_INDENT: usize = 2; +const COMMA: &str = ","; +const SINGLE_SPACE: &str = " "; +const BETWEEN_KEYWORD: &str = "BETWEEN"; +const BETWEEN_SEPARATOR: &str = "AND"; +const VALUES_KEYWORD: &str = "VALUES"; +const DECODE_FUNCTION: &str = "DECODE"; +const CASE_KEYWORD: &str = "CASE"; +const SINGLE_ITEM_CLAUSES: [&str; 4] = ["SELECT", "WHERE", "QUALIFY", "HAVING"]; +const SELECT_MODIFIERS: [&str; 10] = [ + "DISTINCT", "ALL", "TOP", "PERCENT", "WITH", "TIES", "ON", "AS", "STRUCT", "VALUE", +]; +const LEADING_OPERATORS: [&str; 8] = ["+", "-", "*", "/", "%", "||", "AND", "OR"]; +const CHAIN_KEYWORDS: [&str; 2] = ["AND", "OR"]; +const BETWEEN_BOUNDARIES: [&str; 12] = [ + "AND", "OR", "WHEN", "THEN", "ELSE", "WHERE", "ON", "HAVING", "QUALIFY", "SELECT", ",", "NOT", +]; +const ARITHMETIC_OPERATORS: [&str; 5] = ["+", "-", "||", "/", "%"]; +const WINDOW_CLAUSE_KEYWORDS: [&str; 4] = ["ORDER", "ROWS", "RANGE", "GROUPS"]; +const CASE_BRANCH_KEYWORDS: [&str; 3] = ["WHEN", "THEN", "ELSE"]; +const TYPE_PARAMETER_WORDS: [&str; 1] = ["MAX"]; +const TYPE_NAMES: [&str; 30] = [ + "BIGNUMERIC", + "BINARY", + "BIT", + "CHAR", + "CHARACTER", + "DATETIME", + "DATETIME2", + "DATETIMEOFFSET", + "DEC", + "DECIMAL", + "DOUBLE", + "FLOAT", + "INTERVAL", + "NCHAR", + "NUMBER", + "NUMERIC", + "NVARCHAR", + "STRING", + "TEXT", + "TIME", + "TIMESTAMP", + "TIMESTAMPTZ", + "TIMESTAMP_LTZ", + "TIMESTAMP_NTZ", + "TIMESTAMP_TZ", + "TIMETZ", + "VARBINARY", + "VARCHAR", + "BYTES", + "BPCHAR", +]; + +/// Printed SQL whose layout passes only ever rewrite the gaps between tokens. +pub(crate) struct PrintedSql<'a> { + tokens: Vec, + kinds: Vec, + /// The text before each token, plus the text after the last token. + gaps: Vec, + depths: Vec, + partners: Vec>, + /// Gaps that line wrapping must keep: inside VALUES rows and type parameter lists. + locked: Vec, + token_widths: &'a HashMap, + /// Spaces per indent step. + indent: usize, +} + +/// One printed line; end columns are absent when comments or multi-line tokens pin it. +struct Line { + first: usize, + last: usize, + indentation: usize, + end_columns: Option>, +} + +impl<'a> PrintedSql<'a> { + /// Split printed `sql` at its token byte ranges; `None` when it has no tokens. + pub(crate) fn new( + sql: &str, + printed_tokens: &[(Range, TokenType)], + token_widths: &'a HashMap, + ) -> Option { + if printed_tokens.is_empty() { + return None; + } + let mut texts: Vec = Vec::with_capacity(printed_tokens.len()); + let mut gaps: Vec = Vec::with_capacity(printed_tokens.len() + 1); + let mut previous_end = 0_usize; + for (range, _) in printed_tokens { + gaps.push(sql[previous_end..range.start].to_owned()); + texts.push(sql[range.clone()].to_owned()); + previous_end = range.end; + } + gaps.push(sql[previous_end..].to_owned()); + let (depths, partners) = bracket_structure(&texts); + let locked = vec![false; texts.len() + 1]; + Some(Self { + kinds: printed_tokens.iter().map(|(_, kind)| *kind).collect(), + tokens: texts, + gaps, + depths, + partners, + locked, + token_widths, + indent: PARSED_INDENT, + }) + } + + fn len(&self) -> usize { + self.tokens.len() + } + + fn upper(&self, index: usize) -> String { + self.tokens[index].to_ascii_uppercase() + } + + fn is(&self, index: usize, text: &str) -> bool { + self.tokens + .get(index) + .is_some_and(|token| token.eq_ignore_ascii_case(text)) + } + + fn line_ranges(&self, first: usize, last: usize) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut start = first; + for index in first + 1..=last { + if self.gaps[index].contains('\n') { + ranges.push((start, index - 1)); + start = index; + } + } + ranges.push((start, last)); + ranges + } + + /// The first and last token of the printed line holding `index`. + fn line_of(&self, index: usize) -> (usize, usize) { + let first = (1..=index) + .rev() + .find(|&position| self.gaps[position].contains('\n')) + .unwrap_or(0); + let last = (index + 1..self.len()) + .find(|&position| self.gaps[position].contains('\n')) + .map_or(self.len() - 1, |position| position - 1); + (first, last) + } + + fn line(&self, first: usize, last: usize) -> Line { + let indentation = self.indentation(first); + let lead = self.gaps[first].rsplit('\n').next().unwrap_or_default(); + let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); + let commented = !lead.trim().is_empty() + || !trailing.trim().is_empty() + || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()); + let multiline = (first..=last).any(|index| self.tokens[index].contains('\n')); + let end_columns = (!commented && !multiline).then(|| { + let mut column = indentation; + let mut ends: Vec = Vec::with_capacity(last - first + 1); + for index in first..=last { + if index > first { + column += self.gaps[index].chars().count(); + } + column += self.width(index); + ends.push(column); + } + ends + }); + Line { + first, + last, + indentation, + end_columns, + } + } + + /// Indentation of the line that starts at token `first`. + fn indentation(&self, first: usize) -> usize { + self.gaps[first] + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() + } + + fn width(&self, index: usize) -> usize { + let token = &self.tokens[index]; + self.token_widths + .get(&token.to_ascii_uppercase()) + .copied() + .unwrap_or_else(|| token.chars().count()) + } + + /// Whether any gap of the tokens after `first` up to `last` holds a line break. + fn spans_lines(&self, first: usize, last: usize) -> bool { + (first + 1..=last).any(|index| self.gaps[index].contains('\n')) + } + + pub(crate) fn render(&self) -> String { + let mut output = String::new(); + for (gap, token) in self.gaps.iter().zip(&self.tokens) { + output.push_str(gap); + output.push_str(token); + } + output.push_str(&self.gaps[self.tokens.len()]); + output + } +} + +fn bracket_structure(tokens: &[String]) -> (Vec, Vec>) { + let mut depths: Vec = Vec::with_capacity(tokens.len()); + let mut partners: Vec> = vec![None; tokens.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + match token.as_str() { + "(" | "[" | "{" => { + depths.push(open.len()); + open.push(index); + } + ")" | "]" | "}" => { + let opener = open.pop(); + depths.push(open.len()); + if let Some(opener) = opener { + partners[opener] = Some(index); + partners[index] = Some(opener); + } + } + _ => depths.push(open.len()), + } + } + (depths, partners) +} + +fn newline(indentation: usize) -> String { + format!("\n{}", " ".repeat(indentation)) +} + +impl PrintedSql<'_> { + /// Re-indent every line from the parse-tree indent step to `indent` spaces per step. + pub(crate) fn set_indent(&mut self, indent: usize) { + if indent == self.indent { + return; + } + let step = self.indent; + let scale = |spaces: usize| spaces / step * indent + spaces % step; + for gap in &mut self.gaps { + if !gap.contains('\n') { + continue; + } + let comment_start = if gap.contains("/*") { + gap.rfind('\n').unwrap_or(0) + } else { + 0 + }; + let (kept, scaled) = gap.split_at(comment_start); + let mut rebuilt = kept.to_string(); + for (position, segment) in scaled.split('\n').enumerate() { + if position > 0 { + rebuilt.push('\n'); + let spaces = segment.len() - segment.trim_start_matches(' ').len(); + rebuilt.push_str(&" ".repeat(scale(spaces))); + rebuilt.push_str(&segment[spaces..]); + } else { + rebuilt.push_str(segment); + } + } + *gap = rebuilt; + } + self.indent = indent; + } + + /// Rejoin each multi-line expression group, innermost first, whose joined line fits. + pub(crate) fn join_fitting_groups(&mut self, line_width: usize) { + let mut openers: Vec = (0..self.len()) + .filter(|&index| self.tokens[index] == OPEN_PARENTHESIS) + .collect(); + openers.sort_by_key(|&index| std::cmp::Reverse(self.depths[index])); + for opener in openers { + let Some(closer) = self.partners[opener] else { + continue; + }; + if !self.spans_lines(opener, closer) || !self.joinable_group(opener, closer) { + continue; + } + let (first, _) = self.line_of(opener); + let (_, last) = self.line_of(closer); + let indentation = self.indentation(first); + let fits = |start: usize| { + self.joined_width(start, last, indentation) + .is_some_and(|width| width <= line_width) + }; + let start = if fits(first) { + first + } else if let Some(operator) = self.leading_operator_before(first, opener) + && fits(operator) + { + operator + } else { + continue; + }; + for index in opener + 1..=closer { + if self.gaps[index].contains('\n') { + self.gaps[index] = self.joined_gap(index).to_string(); + } + } + if start != first { + self.gaps[start] = newline(indentation); + } + } + } + + /// The last binary operator on the line from `first` that applies to the group at `opener`. + fn leading_operator_before(&self, first: usize, opener: usize) -> Option { + (first + 1..opener).rev().find(|&index| { + self.depths[index] == self.depths[opener] + && LEADING_OPERATORS.contains(&self.upper(index).as_str()) + && self.gaps[index] == SINGLE_SPACE + && self.ends_operand(index - 1) + }) + } + + /// Move a comma that follows a line comment to the end of the commented line. + pub(crate) fn attach_commas_before_comments(&mut self) { + for index in 1..self.len().saturating_sub(1) { + if self.tokens[index] != COMMA { + continue; + } + let Some((comment, rest)) = self.gaps[index].split_once('\n') else { + continue; + }; + if !comment.trim_start().starts_with("--") || !rest.trim().is_empty() { + continue; + } + let (comment, rest) = (comment.to_string(), rest.to_string()); + let next = self.gaps[index + 1].trim_start_matches(' ').to_string(); + self.gaps[index] = String::new(); + self.gaps[index + 1] = if next.starts_with('\n') { + comment + &next + } else { + format!("{comment}\n{rest}{next}") + }; + } + } + + fn joinable_group(&self, opener: usize, closer: usize) -> bool { + (opener + 1..closer).all(|index| { + !(self.kinds[index] == TokenType::Select + || self.is(index, VALUES_KEYWORD) + || (self.is(index, CASE_KEYWORD) + && self + .case_end(index) + .is_none_or(|end| self.spans_lines(index, end)))) + }) + } + + /// The width of tokens `first..=last` on one line at `indentation`; `None` when comments pin. + fn joined_width(&self, first: usize, last: usize, indentation: usize) -> Option { + let trailing = self.gaps[last + 1].split('\n').next().unwrap_or_default(); + let pinned = !trailing.trim().is_empty() + || (first + 1..=last).any(|index| !self.gaps[index].trim().is_empty()) + || (first..=last).any(|index| self.tokens[index].contains('\n')); + if pinned { + return None; + } + let mut width = indentation; + for index in first..=last { + if index > first { + width += if self.gaps[index].contains('\n') { + self.joined_gap(index).len() + } else { + self.gaps[index].chars().count() + }; + } + width += self.width(index); + } + Some(width) + } + + /// The gap before `index` once its line break is removed. + fn joined_gap(&self, index: usize) -> &'static str { + if self.tokens[index - 1] == OPEN_PARENTHESIS + || matches!(self.tokens[index].as_str(), ")" | ",") + { + "" + } else { + SINGLE_SPACE + } + } + + /// Apply the layout rules that only move existing line breaks before wrapping. + pub(crate) fn arrange_clauses(&mut self, line_width: usize) { + for index in 0..self.len() { + if self.is(index, DECODE_FUNCTION) { + self.pair_decode_arguments(index); + } + } + for index in 1..self.len() { + if CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) { + self.join_chain_break_in_group(index); + } + } + for index in 0..self.len() { + if self.is(index, VALUES_KEYWORD) { + self.break_values_rows_unless_fitting(index, line_width); + } + } + for index in 0..self.len() { + if self.kinds[index] == TokenType::With { + self.separate_common_table_expressions(index); + } + } + for index in 0..self.len() { + if SINGLE_ITEM_CLAUSES.contains(&self.upper(index).as_str()) { + self.join_single_line_body(index, line_width); + } + } + } + + /// Start the next line with any binary operator whose operand spans several lines. + pub(crate) fn lead_operators_of_multiline_operands(&mut self) { + for index in 1..self.len().saturating_sub(1) { + if !LEADING_OPERATORS.contains(&self.upper(index).as_str()) + || self.gaps[index] != SINGLE_SPACE + || self.gaps[index + 1].contains('\n') + || !self.ends_operand(index - 1) + || (self.is(index, "AND") && self.follows_between(index)) + { + continue; + } + let multiline = self + .left_operand_start(index - 1) + .is_some_and(|start| self.spans_lines(start, index - 1)) + || self + .right_operand_end(index + 1) + .is_some_and(|end| self.spans_lines(index + 1, end)); + if multiline { + let (first, _) = self.line_of(index); + self.gaps[index] = newline(self.indentation(first)); + } + } + } + + /// Put the first DECODE argument on its own line and each search/result pair on one line. + fn pair_decode_arguments(&mut self, name: usize) { + let opener = name + 1; + if !self.is(opener, OPEN_PARENTHESIS) || (name > 0 && self.is(name - 1, ".")) { + return; + } + let Some(closer) = self.partners[opener] else { + return; + }; + if !self.gaps[opener + 1].contains('\n') { + return; + } + let mut argument = 0_usize; + for index in opener + 2..closer { + if self.depths[index] != self.depths[opener] + 1 || self.tokens[index - 1] != COMMA { + continue; + } + argument += 1; + if argument.is_multiple_of(2) && self.gaps[index].trim().is_empty() { + self.gaps[index] = SINGLE_SPACE.to_string(); + } + } + } + + /// Join an AND or OR line break inside a bracket group whose arguments share its line. + fn join_chain_break_in_group(&mut self, index: usize) { + if !self.gaps[index].contains('\n') || !self.gaps[index].trim().is_empty() { + return; + } + let depth = self.depths[index]; + let Some(opener) = (0..index) + .rev() + .find(|&previous| self.depths[previous] < depth) + else { + return; + }; + if self.tokens[opener] == OPEN_PARENTHESIS && !self.spans_lines(opener, index - 1) { + self.gaps[index] = SINGLE_SPACE.to_string(); + } + } + + /// Start every row of a VALUES list on its own line unless the whole list fits on its line. + fn break_values_rows_unless_fitting(&mut self, values: usize, line_width: usize) { + let rows = self.values_rows(values); + let Some(&(_, last_closer)) = rows.last() else { + return; + }; + let (first, last) = self.line_of(values); + let fits = !self.spans_lines(values, last_closer) + && self + .line(first, last) + .end_columns + .is_some_and(|ends| ends.last().is_some_and(|end| *end <= line_width)); + if fits { + return; + } + let mut indentation = self.indentation(first); + if self.gaps[values].contains('\n') { + indentation = self.indentation(values); + } else if values > 0 + && self.tokens[values - 1] == OPEN_PARENTHESIS + && let Some(closer) = self.partners[values - 1] + { + self.set_break(values, &newline(indentation + self.indent)); + self.set_break(closer, &newline(indentation)); + indentation += self.indent; + } + let row_line = newline(indentation + self.indent); + for (opener, _) in rows { + self.set_break(opener, &row_line); + } + } + + /// Separate the CTEs of the WITH at `with`, and the statement after them, by one blank line. + fn separate_common_table_expressions(&mut self, with: usize) { + let (first, _) = self.line_of(with); + let indentation = self.indentation(first); + let mut item = with + 1; + if self.is(item, "RECURSIVE") { + item += 1; + } + while let Some(end) = self.common_table_expression_end(item) { + let next = end + 1; + if self.is(next, ",") { + self.blank_line_before(next + 1, indentation); + item = next + 1; + continue; + } + if next < self.len() && !self.is(next, ";") && !self.is(next, ")") { + self.blank_line_before(next, indentation); + } + break; + } + } + + /// The last token of the CTE that starts at `item`. + fn common_table_expression_end(&self, item: usize) -> Option { + let name = self.tokens.get(item)?; + if !matches!( + self.kinds[item], + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ) && !name.chars().all(|c| c.is_alphanumeric() || c == '_') + { + return None; + } + let mut index = item + 1; + if self.is(index, OPEN_PARENTHESIS) { + index = self.partners[index]? + 1; + } + if !self.is(index, "AS") { + return None; + } + index += 1; + if self.is(index, "NOT") { + index += 1; + } + if self.is(index, "MATERIALIZED") { + index += 1; + } + if !self.is(index, OPEN_PARENTHESIS) { + return None; + } + self.partners[index] + } + + /// Replace the line break before `index` with exactly one blank line, keeping comments. + fn blank_line_before(&mut self, index: usize, indentation: usize) { + let gap = &self.gaps[index]; + let (trailing, rest) = gap.split_once('\n').unwrap_or((gap.as_str(), "")); + let rest = rest.trim_start(); + let margin = " ".repeat(indentation); + self.gaps[index] = if rest.is_empty() { + format!("{}\n\n{margin}", trailing.trim_end()) + } else { + format!("{}\n\n{margin}{rest}", trailing.trim_end()) + }; + } + + /// Move a clause body that is one line under its clause keyword onto the keyword line. + fn join_single_line_body(&mut self, clause: usize, line_width: usize) { + let (first, last) = self.line_of(clause); + let body = last + 1; + let is_select = self.is(clause, "SELECT"); + let modifiers_only = (clause + 1..=last).all(|index| { + SELECT_MODIFIERS.contains(&self.upper(index).as_str()) + || self.kinds[index] == TokenType::Number + || matches!(self.tokens[index].as_str(), "(" | ")" | ",") + || self.depths[index] > self.depths[clause] + }); + if body >= self.len() + || clause != first + || (!is_select && clause != last) + || (is_select && !modifiers_only) + || !self.gaps[body].contains('\n') + || !self.gaps[body].trim().is_empty() + { + return; + } + let clause_indentation = self.indentation(first); + if self.indentation(body) <= clause_indentation { + return; + } + let (_, body_last) = self.line_of(body); + if body_last + 1 < self.len() && self.indentation(body_last + 1) > clause_indentation { + return; + } + let pinned = (body + 1..=body_last).any(|index| !self.gaps[index].trim().is_empty()) + || (first..=body_last).any(|index| self.tokens[index].contains('\n')); + if pinned || (!is_select && self.has_top_level_chain(body, body_last)) { + return; + } + let mut width = clause_indentation; + for index in first..=body_last { + if index > first { + width += if index == body { + 1 + } else { + self.gaps[index].chars().count() + }; + } + width += self.width(index); + } + width += self.gaps[body_last + 1] + .split('\n') + .next() + .unwrap_or_default() + .chars() + .count(); + if width <= line_width { + self.gaps[body] = SINGLE_SPACE.to_string(); + } + } + + fn has_top_level_chain(&self, first: usize, last: usize) -> bool { + let depth = (first..=last).map(|index| self.depths[index]).min(); + (first..=last).any(|index| { + Some(self.depths[index]) == depth + && CHAIN_KEYWORDS.contains(&self.upper(index).as_str()) + && !(self.is(index, "AND") && self.follows_between(index)) + }) + } + + /// Whether the AND at `index` separates the bounds of a BETWEEN. + fn follows_between(&self, index: usize) -> bool { + let depth = self.depths[index]; + for previous in (0..index).rev() { + if self.depths[previous] < depth { + return false; + } + if self.depths[previous] > depth { + continue; + } + let upper = self.upper(previous); + if upper == BETWEEN_KEYWORD { + return true; + } + if BETWEEN_BOUNDARIES.contains(&upper.as_str()) { + return false; + } + } + false + } + + fn ends_operand(&self, index: usize) -> bool { + matches!(self.tokens[index].as_str(), ")" | "]") + || self.is(index, "END") + || matches!( + self.kinds[index], + TokenType::Var + | TokenType::Identifier + | TokenType::QuotedIdentifier + | TokenType::Number + | TokenType::String + ) + } + + /// The first token of a bracket group or CASE expression that ends at `last`. + fn left_operand_start(&self, last: usize) -> Option { + if matches!(self.tokens[last].as_str(), ")" | "]") { + return self.partners[last]; + } + if !self.is(last, "END") { + return None; + } + let depth = self.depths[last]; + let mut nested = 0_usize; + for previous in (0..last).rev() { + if self.depths[previous] != depth { + continue; + } + if self.is(previous, "END") { + nested += 1; + } else if self.is(previous, "CASE") { + if nested == 0 { + return Some(previous); + } + nested -= 1; + } + } + None + } + + /// The last token of a bracket group, call, or CASE expression that starts at `first`. + fn right_operand_end(&self, first: usize) -> Option { + if self.tokens[first] == OPEN_PARENTHESIS { + return self.partners[first]; + } + let word = self.tokens[first] + .chars() + .all(|character| character.is_alphanumeric() || character == '_'); + if word && self.is(first + 1, OPEN_PARENTHESIS) { + return self.partners[first + 1]; + } + if !self.is(first, CASE_KEYWORD) { + return None; + } + self.case_end(first) + } +} + +/// Re-break every line of `printed` that is longer than the line width and can be shortened. +pub(crate) fn wrap_lines(printed: &mut PrintedSql<'_>, line_width: usize) { + printed.lock_unbreakable_groups(); + let mut pending: Vec<(usize, usize)> = printed.line_ranges(0, printed.len() - 1); + pending.reverse(); + while let Some((first, last)) = pending.pop() { + let line = printed.line(first, last); + if !overflows(&line, line_width) || !printed.break_line(&line) { + continue; + } + let mut lines = printed.line_ranges(first, last); + lines.reverse(); + pending.extend(lines); + } +} + +fn overflows(line: &Line, line_width: usize) -> bool { + line.end_columns + .as_ref() + .is_some_and(|ends| ends.last().is_some_and(|end| *end > line_width)) +} + +impl PrintedSql<'_> { + /// Lock the inside of VALUES rows and of type parameter lists such as `NUMBER(38, 2)`. + fn lock_unbreakable_groups(&mut self) { + for index in 0..self.len() { + if self.is(index, VALUES_KEYWORD) { + for (opener, closer) in self.values_rows(index) { + self.lock(opener, closer); + } + } else if let Some(closer) = self.type_parameters_closer(index) { + self.lock(index - 1, closer); + } + } + } + + fn lock(&mut self, first: usize, last: usize) { + for gap in first + 1..=last { + self.locked[gap] = true; + } + } + + /// The bracketed rows that directly follow the VALUES keyword at `values`. + fn values_rows(&self, values: usize) -> Vec<(usize, usize)> { + let mut rows: Vec<(usize, usize)> = Vec::new(); + let mut opener = values + 1; + while opener < self.len() && self.tokens[opener] == OPEN_PARENTHESIS { + let Some(closer) = self.partners[opener] else { + break; + }; + rows.push((opener, closer)); + if !(self.is(closer + 1, COMMA) && self.is(closer + 2, OPEN_PARENTHESIS)) { + break; + } + opener = closer + 2; + } + rows + } + + /// The closer of a type parameter list opened at `opener`, such as `(38, 2)` of `NUMBER`. + fn type_parameters_closer(&self, opener: usize) -> Option { + if opener < 1 || self.tokens[opener] != OPEN_PARENTHESIS { + return None; + } + let closer = self.partners[opener]?; + let name = &self.tokens[opener - 1]; + if closer == opener + 1 || !name.chars().all(|c| c.is_alphanumeric() || c == '_') { + return None; + } + let type_position = opener > 1 && (self.is(opener - 2, "::") || self.is(opener - 2, "AS")); + if !type_position && !TYPE_NAMES.contains(&name.to_ascii_uppercase().as_str()) { + return None; + } + let parameters = (opener + 1..closer).all(|index| { + let token = &self.tokens[index]; + token == COMMA + || is_digits(token) + || TYPE_PARAMETER_WORDS.contains(&token.to_ascii_uppercase().as_str()) + }); + parameters.then_some(closer) + } + + /// Apply the outermost applicable break: chains, VALUES rows, bracket groups, then commas. + fn break_line(&mut self, line: &Line) -> bool { + let values_row = self.tokens[line.first] == OPEN_PARENTHESIS && self.locked[line.first + 1]; + !values_row + && (self.break_before(line, &CHAIN_KEYWORDS, 0) + || self.break_before(line, &ARITHMETIC_OPERATORS, self.indent) + || self.break_values_rows(line) + || self.explode_group(line) + || self.break_after_commas(line)) + } + + /// Put each row of a VALUES list that starts this line on its own line. + fn break_values_rows(&mut self, line: &Line) -> bool { + let depth = self.line_depth(line); + let Some(values) = (line.first..=line.last).find(|&index| { + self.is(index, VALUES_KEYWORD) + && (self.depths[index] == depth || index == line.first) + && !self.locked[index + 1] + }) else { + return false; + }; + let rows = self.values_rows(values); + if rows.is_empty() + || rows + .iter() + .all(|(opener, _)| self.gaps[*opener].contains('\n')) + { + return false; + } + let row_line = newline(line.indentation + self.indent); + for (opener, _) in rows { + self.gaps[opener] = row_line.clone(); + } + true + } + + /// Put the arguments or CASE branches of the outermost, widest group on their own lines. + fn explode_group(&mut self, line: &Line) -> bool { + let Some(ends) = line.end_columns.as_ref() else { + return false; + }; + let start_column = |index: usize| { + if index == line.first { + line.indentation + } else { + ends[index - 1 - line.first] + } + }; + let line_end = ends[ends.len() - 1]; + let mut chosen: Option<(usize, usize, usize)> = None; + for opener in line.first..=line.last { + let Some(closer) = self.group_closer(opener, line) else { + continue; + }; + let end_column = if closer > line.last { + line_end + } else { + ends[closer - line.first] + }; + let width = end_column - start_column(opener); + let better = chosen.is_none_or(|(best, _, best_width)| { + self.depths[opener] < self.depths[best] + || (self.depths[opener] == self.depths[best] && width >= best_width) + }); + if better { + chosen = Some((opener, closer, width)); + } + } + let Some((opener, closer, _)) = chosen else { + return false; + }; + if closer > line.last { + self.indent_lines(opener + 1, closer - 1, self.indent); + } + let inner = newline(line.indentation + self.indent); + if self.is(opener, CASE_KEYWORD) { + self.explode_case(opener, closer, line.indentation); + return true; + } + self.set_break(opener + 1, &inner); + self.set_break(closer, &newline(line.indentation)); + if self.is(opener + 1, VALUES_KEYWORD) { + return true; + } + let window = opener > 0 && self.is(opener - 1, "OVER"); + let decode = opener > 0 && self.is(opener - 1, DECODE_FUNCTION); + let mut argument = 0_usize; + for index in opener + 2..closer { + if self.depths[index] != self.depths[opener] + 1 { + continue; + } + if window { + if WINDOW_CLAUSE_KEYWORDS.contains(&self.upper(index).as_str()) { + self.set_break(index, &inner); + } + } else if self.tokens[index - 1] == COMMA { + argument += 1; + if !decode || !argument.is_multiple_of(2) { + self.set_break(index, &inner); + } + } + } + true + } + + /// The closer of an explodable group opened at `opener` on `line`. + fn group_closer(&self, opener: usize, line: &Line) -> Option { + if self.is(opener, CASE_KEYWORD) { + let end = self.case_end(opener)?; + return (end <= line.last && end > opener + 2).then_some(end); + } + let closer = self.partners[opener]?; + let explodable = closer > opener + 2 + && matches!(self.tokens[opener].as_str(), "(" | "[" | "{") + && !self.locked[opener + 1] + && (closer <= line.last || opener < line.last); + explodable.then_some(closer) + } + + /// The END of the CASE expression at `case`. + fn case_end(&self, case: usize) -> Option { + let depth = self.depths[case]; + let mut nested = 0_usize; + for index in case + 1..self.len() { + if self.depths[index] < depth { + return None; + } + if self.depths[index] != depth { + continue; + } + if self.is(index, CASE_KEYWORD) { + nested += 1; + } else if self.is(index, "END") { + if nested == 0 { + return Some(index); + } + nested -= 1; + } + } + None + } + + /// Put each WHEN, THEN and ELSE of a one-line CASE on its own line and END under CASE. + fn explode_case(&mut self, case: usize, end: usize, indentation: usize) { + let inner = newline(indentation + self.indent); + let depth = self.depths[case]; + let mut nested = 0_usize; + for index in case + 1..end { + if self.depths[index] != depth { + continue; + } + if self.is(index, CASE_KEYWORD) { + nested += 1; + } else if self.is(index, "END") { + nested -= 1; + } else if nested == 0 && CASE_BRANCH_KEYWORDS.contains(&self.upper(index).as_str()) { + self.set_break(index, &inner); + } + } + self.set_break(end, &newline(indentation)); + } + + /// Replace a whitespace-only gap; comments keep their gaps. + fn set_break(&mut self, index: usize, gap: &str) { + if self.gaps[index].trim().is_empty() { + self.gaps[index] = gap.to_string(); + } + } + + /// Indent lines starting within `first..=last` by `extra`, except gaps with block comments. + fn indent_lines(&mut self, first: usize, last: usize, extra: usize) { + let margin = format!("\n{}", " ".repeat(extra)); + for index in first..=last { + if self.gaps[index].contains('\n') && !self.gaps[index].contains("/*") { + self.gaps[index] = self.gaps[index].replace('\n', &margin); + } + } + } + + /// Break before every operator of `operators` at the line's outermost depth. + fn break_before(&mut self, line: &Line, operators: &[&str], extra_indent: usize) -> bool { + let depth = self.line_depth(line); + let mut between = false; + let mut breaks: Vec = Vec::new(); + for index in line.first..=line.last { + if self.depths[index] != depth || self.locked[index] { + continue; + } + let upper = self.upper(index); + if upper == BETWEEN_KEYWORD { + between = true; + continue; + } + if upper == BETWEEN_SEPARATOR && between { + between = false; + continue; + } + if index > line.first + && operators.contains(&upper.as_str()) + && self.gaps[index] == SINGLE_SPACE + && ends_operand(&self.tokens[index - 1]) + { + breaks.push(index); + } + } + let indentation = newline(line.indentation + extra_indent); + for index in &breaks { + self.gaps[*index] = indentation.clone(); + } + !breaks.is_empty() + } + + /// Break after every comma at the outermost comma depth of the line. + fn break_after_commas(&mut self, line: &Line) -> bool { + let Some(depth) = (line.first..line.last) + .filter(|&index| self.tokens[index] == COMMA && !self.locked[index + 1]) + .map(|index| self.depths[index]) + .min() + else { + return false; + }; + let indentation = newline(line.indentation + self.indent); + let mut changed = false; + for index in line.first..line.last { + if self.depths[index] == depth && self.tokens[index] == COMMA && !self.locked[index + 1] + { + self.gaps[index + 1] = indentation.clone(); + changed = true; + } + } + changed + } + + fn line_depth(&self, line: &Line) -> usize { + (line.first..=line.last) + .map(|index| self.depths[index]) + .min() + .unwrap_or_default() + } +} + +fn is_digits(token: &str) -> bool { + token.chars().all(|character| character.is_ascii_digit()) +} + +fn ends_operand(token: &str) -> bool { + token.chars().last().is_some_and(|character| { + character.is_alphanumeric() || matches!(character, '_' | '\'' | '"' | '`' | ')' | ']') + }) +} diff --git a/crates/sqlquack-format/src/format/_helpers/token_layout.rs b/crates/sqlquack-format/src/format/_helpers/token_layout.rs new file mode 100644 index 000000000..454c0f9d2 --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/token_layout.rs @@ -0,0 +1,597 @@ +//! Print authored SQL tokens with whitespace taken from the layout guide, and check the result. + +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Range; + +use sqlquack_syntax::tokens::models::TokenType; + +use crate::format::_helpers::guide::GuideLayout; +use crate::format::_helpers::lexemes::{ + Comment, Lexeme, is_keyword, is_string_literal, is_unquoted_word, +}; +use crate::format::constants::{ + ALIGNMENT_FAILURE, CLOSE_PARENTHESIS, COMMENT_ATTACHMENT_FAILURE, OPEN_PARENTHESIS, + STRING_BREAK_FAILURE, TOKEN_PRESERVATION_FAILURE, +}; + +const RESYNC_WINDOW: usize = 48; +const RESYNC_CONFIRMATION: usize = 1; +const NO_SPACE_BEFORE: [&str; 7] = [",", ")", "]", ".", ";", "::", ":"]; +const NO_SPACE_AFTER: [&str; 5] = ["(", "[", ".", "::", ":"]; +const STRUCTURAL_PUNCTUATION: [&str; 6] = ["(", ")", "[", "]", ",", ";"]; + +/// One authored or guide token with its text and alignment key. +struct Keyed { + start: usize, + end: usize, + raw: String, + key: Key, + /// An unquoted authored word outside the name positions that keep reserved words exact. + recasable: bool, + /// An unquoted word the tokenizer types as a keyword. + keyword: bool, +} + +/// How tokens compare: words ignoring ASCII case, punctuation by type, the rest by text. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Key { + Word, + Punctuation(TokenType), + Raw, +} + +impl Keyed { + fn new(lexeme: &Lexeme, recasable: bool) -> Self { + let raw = &lexeme.raw; + let key = if is_unquoted_word(raw) { + Key::Word + } else if raw + .chars() + .all(|character| character.is_ascii_punctuation()) + { + Key::Punctuation(lexeme.kind) + } else { + Key::Raw + }; + Self { + start: lexeme.start, + end: lexeme.end, + raw: raw.clone(), + key, + recasable, + keyword: is_keyword(lexeme), + } + } + + fn same_key(&self, other: &Keyed) -> bool { + self.key == other.key + && match self.key { + Key::Word => self.raw.eq_ignore_ascii_case(&other.raw), + Key::Punctuation(_) => true, + Key::Raw => self.raw == other.raw, + } + } +} + +/// The guide's tokens and text, read once for every print against it. +pub(crate) struct Oracle { + generated: Vec, + text: String, + /// Token ranges, in order and apart, whose authored text is printed byte for byte. + pub(crate) verbatim: Vec<(usize, usize)>, +} + +impl Oracle { + pub(crate) fn new(guide: GuideLayout) -> Self { + let generated: Vec = guide + .tokens + .iter() + .map(|token| Keyed::new(token, false)) + .collect(); + Self { + generated, + text: guide.text, + verbatim: guide.verbatim, + } + } +} + +/// Authored tokens printed against the guide, with what the token check needs about them. +pub(crate) struct AuthoredPrint { + pub(crate) sql: String, + /// The byte range and type of each printed token; comments and whitespace lie between them. + pub(crate) tokens: Vec<(Range, TokenType)>, + pub(crate) recases: BTreeMap, +} + +/// Comments attached to the authored token they precede or follow. +#[derive(Default)] +struct Attached<'a> { + leading: Vec<&'a Comment>, + trailing: Vec<&'a Comment>, +} + +/// Authored SQL with its tokens and the comments between them. +pub(crate) struct AuthoredSql<'a> { + pub(crate) sql: &'a str, + pub(crate) lexemes: &'a [Lexeme], + pub(crate) comments: &'a [Comment], +} + +fn keyed_authored(lexemes: &[Lexeme]) -> Vec { + lexemes + .iter() + .enumerate() + .map(|(index, lexeme)| { + let name = index > 0 + && (matches!(lexemes[index - 1].raw.as_str(), "." | ":" | "@" | "$") + || lexemes[index - 1].raw.eq_ignore_ascii_case("AS")); + Keyed::new(lexeme, !name && is_unquoted_word(&lexeme.raw)) + }) + .collect() +} + +/// Print every authored token in order, taking only whitespace and keyword case from `oracle`. +pub(crate) fn print_authored_tokens( + source: &AuthoredSql<'_>, + oracle: &Oracle, +) -> Result { + let AuthoredSql { + sql, + lexemes, + comments, + } = *source; + let authored = keyed_authored(lexemes); + let generated = &oracle.generated; + let alignment = align(&authored, generated)?; + let separators = separators(&authored, generated, &alignment, &oracle.text)?; + let attached = attach_comments(lexemes, comments); + let mut output = String::with_capacity(sql.len() + sql.len() / 4); + let string_breaks = string_breaks(sql, lexemes); + let mut recases: BTreeMap = BTreeMap::new(); + let mut printed_tokens: Vec<(Range, TokenType)> = Vec::with_capacity(lexemes.len()); + for (index, lexeme) in authored.iter().enumerate() { + let forced_break = string_breaks.contains(&index) && !separators[index].contains('\n'); + let forced: String; + let separator: &str = if forced_break { + forced = format!("\n{}", current_indentation(&output)); + &forced + } else { + &separators[index] + }; + let leading = &attached[index].leading; + if leading.is_empty() { + output.push_str(separator); + } else if index == 0 || separator.contains('\n') { + output.push_str(separator); + let indentation = current_indentation(&output); + for comment in leading { + output.push_str(&comment.text); + output.push('\n'); + output.push_str(&indentation); + } + } else { + let indentation = current_indentation(&output); + for comment in leading { + output.push('\n'); + output.push_str(&indentation); + output.push_str(&comment.text); + } + output.push('\n'); + output.push_str(&indentation); + } + let guide = alignment[index].map(|at| &generated[at]); + let start = output.len(); + match guide.filter(|guide| confirms_keyword(lexeme, guide)) { + Some(guide) => { + recases.insert(index, guide.raw.clone()); + output.push_str(&guide.raw); + } + None => output.push_str(&lexeme.raw), + } + printed_tokens.push((start..output.len(), lexemes[index].kind)); + let trailing = &attached[index].trailing; + for (position, comment) in trailing.iter().enumerate() { + if position == 0 || !trailing[position - 1].line { + output.push(' '); + } + output.push_str(&comment.text); + if !comment.line { + continue; + } + let next_starts_line = if position + 1 < trailing.len() { + false + } else { + separators + .get(index + 1) + .is_none_or(|next| next.starts_with('\n')) + }; + if !next_starts_line { + output.push('\n'); + if position + 1 < trailing.len() { + output.push_str(¤t_indentation(&output)); + } + } + } + } + Ok(AuthoredPrint { + sql: output, + tokens: printed_tokens, + recases, + }) +} + +/// Refuse output that changes anything but layout and the recorded keyword recases. +pub(crate) fn verify_printed_tokens( + before: &AuthoredSql<'_>, + after: &AuthoredSql<'_>, + recases: &BTreeMap, +) -> Result<(), String> { + if before.lexemes.len() != after.lexemes.len() { + return Err(TOKEN_PRESERVATION_FAILURE.to_string()); + } + for (index, (authored, printed)) in before.lexemes.iter().zip(after.lexemes).enumerate() { + if authored.raw == printed.raw { + continue; + } + let allowed = recases.get(&index).is_some_and(|recase| { + *recase == printed.raw + && is_keyword(printed) + && printed.raw == printed.raw.to_ascii_uppercase() + && authored.raw.eq_ignore_ascii_case(&printed.raw) + }); + if !allowed { + return Err(TOKEN_PRESERVATION_FAILURE.to_string()); + } + } + let attached = comments_stay_attached( + &comment_positions(before.comments, before.lexemes), + &comment_positions(after.comments, after.lexemes), + after.lexemes, + ); + if !attached { + return Err(COMMENT_ATTACHMENT_FAILURE.to_string()); + } + if !string_breaks(before.sql, before.lexemes) + .is_subset(&string_breaks(after.sql, after.lexemes)) + { + return Err(STRING_BREAK_FAILURE.to_string()); + } + Ok(()) +} + +/// Whether every comment keeps its place, allowing a line comment to move just past a comma. +fn comments_stay_attached( + before: &[(&str, usize)], + after: &[(&str, usize)], + after_lexemes: &[Lexeme], +) -> bool { + before.len() == after.len() + && before.iter().zip(after).all( + |((before_text, before_position), (after_text, after_position))| { + before_text == after_text + && (before_position == after_position + || (*after_position == before_position + 1 + && before_text.starts_with("--") + && after_lexemes + .get(*before_position) + .is_some_and(|lexeme| lexeme.kind == TokenType::Comma))) + }, + ) +} + +fn comment_positions<'a>(comments: &'a [Comment], lexemes: &[Lexeme]) -> Vec<(&'a str, usize)> { + comments + .iter() + .map(|comment| { + let preceding = lexemes.partition_point(|lexeme| lexeme.end <= comment.start); + (comment.text.as_str(), preceding) + }) + .collect() +} + +/// Indices of string literals that follow another string literal across a line break. +fn string_breaks(sql: &str, lexemes: &[Lexeme]) -> BTreeSet { + (1..lexemes.len()) + .filter(|&index| { + is_string_literal(lexemes[index - 1].kind) + && is_string_literal(lexemes[index].kind) + && sql[lexemes[index - 1].end..lexemes[index].start].contains('\n') + }) + .collect() +} + +/// Map authored tokens to matching guide tokens; a closer only matches its opener's partner. +fn align(authored: &[Keyed], generated: &[Keyed]) -> Result>, String> { + let mut state = Alignment { + authored, + generated, + authored_partners: partners(authored), + generated_partners: partners(generated), + forward: vec![None; authored.len()], + backward: vec![None; generated.len()], + }; + let (mut left, mut right) = (0_usize, 0_usize); + while left < authored.len() && right < generated.len() { + if state.matches(left, right) { + state.forward[left] = Some(right); + state.backward[right] = Some(left); + left += 1; + right += 1; + continue; + } + let (skip_left, skip_right) = state + .resync(left, right) + .ok_or_else(|| ALIGNMENT_FAILURE.to_string())?; + left += skip_left; + right += skip_right; + } + Ok(state.forward) +} + +struct Alignment<'a> { + authored: &'a [Keyed], + generated: &'a [Keyed], + authored_partners: Vec>, + generated_partners: Vec>, + forward: Vec>, + backward: Vec>, +} + +impl Alignment<'_> { + fn matches(&self, left: usize, right: usize) -> bool { + if !self.authored[left].same_key(&self.generated[right]) { + return false; + } + if !is_closer(&self.authored[left].raw) { + return true; + } + match (self.authored_partners[left], self.generated_partners[right]) { + (Some(opener), Some(generated_opener)) => match self.forward[opener] { + Some(aligned) => aligned == generated_opener, + None => self.backward[generated_opener].is_none(), + }, + (None, None) => true, + _ => false, + } + } + + fn resync(&self, left: usize, right: usize) -> Option<(usize, usize)> { + let remaining_left = self.authored.len() - left; + let remaining_right = self.generated.len() - right; + for cost in 1..=(2 * RESYNC_WINDOW) { + for skip_left in 0..=cost.min(RESYNC_WINDOW) { + let skip_right = cost - skip_left; + if skip_right > RESYNC_WINDOW { + continue; + } + if skip_left >= remaining_left || skip_right >= remaining_right { + if skip_left >= remaining_left && skip_right >= remaining_right { + return Some((remaining_left, remaining_right)); + } + continue; + } + if !self.matches(left + skip_left, right + skip_right) { + continue; + } + let anchor = &self.authored[left + skip_left]; + let confirmations = if matches!(anchor.key, Key::Punctuation(_)) { + RESYNC_CONFIRMATION + } else { + 0 + }; + let confirmed = (1..=confirmations).all(|offset| { + match ( + self.authored.get(left + skip_left + offset), + self.generated.get(right + skip_right + offset), + ) { + (Some(before), Some(after)) => before.same_key(after), + _ => true, + } + }); + if confirmed { + return Some((skip_left, skip_right)); + } + } + } + None + } +} + +fn is_closer(raw: &str) -> bool { + matches!(raw, ")" | "]" | "}") +} + +fn partners(tokens: &[Keyed]) -> Vec> { + let mut partners: Vec> = vec![None; tokens.len()]; + let mut open: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + match token.raw.as_str() { + "(" | "[" | "{" => open.push(index), + ")" | "]" | "}" => { + if let Some(opener) = open.pop() { + partners[opener] = Some(index); + partners[index] = Some(opener); + } + } + _ => {} + } + } + partners +} + +fn separators( + authored: &[Keyed], + generated: &[Keyed], + alignment: &[Option], + oracle: &str, +) -> Result, String> { + let mut separators: Vec = vec![String::new(); authored.len()]; + let anchors: Vec = (0..authored.len()) + .filter(|&index| alignment[index].is_some()) + .collect(); + if anchors.is_empty() { + return Err(ALIGNMENT_FAILURE.to_string()); + } + let gap = |left: usize, right: usize| -> String { + oracle[generated[left].end..generated[right].start].to_string() + }; + for index in 1..anchors[0] { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + if anchors[0] > 0 { + separators[anchors[0]] = + default_space(&authored[anchors[0] - 1], &authored[anchors[0]]).to_string(); + } + for pair in anchors.windows(2) { + let (previous, next) = (pair[0], pair[1]); + let (previous_at, next_at) = ( + alignment[previous].ok_or(ALIGNMENT_FAILURE)?, + alignment[next].ok_or(ALIGNMENT_FAILURE)?, + ); + let merged = merged_gap( + &(previous_at..next_at) + .map(|at| gap(at, at + 1)) + .collect::>(), + ); + if previous + 1 == next { + separators[next] = merged; + continue; + } + for index in previous + 2..next { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + let first = previous + 1; + let last = next - 1; + let substitution = next_at > previous_at + 1; + separators[first] = if default_space(&authored[previous], &authored[first]).is_empty() { + String::new() + } else if substitution { + gap(previous_at, previous_at + 1) + } else { + " ".to_string() + }; + separators[next] = if substitution { + let oracle_gap = (previous_at + 1..next_at) + .rev() + .find(|&at| generated[at].same_key(&authored[last])) + .map_or_else(|| gap(next_at - 1, next_at), |at| gap(at, at + 1)); + let unspaced = default_space(&authored[last], &authored[next]).is_empty(); + if unspaced && !oracle_gap.contains('\n') { + String::new() + } else { + oracle_gap + } + } else if merged.contains('\n') { + merged + } else { + default_space(&authored[last], &authored[next]).to_string() + }; + } + let last_anchor = anchors[anchors.len() - 1]; + for index in last_anchor + 1..authored.len() { + separators[index] = default_space(&authored[index - 1], &authored[index]).to_string(); + } + let last_at = alignment[last_anchor].ok_or(ALIGNMENT_FAILURE)?; + if last_anchor + 1 < authored.len() + && last_at + 1 < generated.len() + && !separators[last_anchor + 1].is_empty() + { + separators[last_anchor + 1] = gap(last_at, last_at + 1); + } + for index in 1..authored.len() { + let adjacent_in_source = authored[index - 1].end == authored[index].start; + let oracle_pair = matches!( + (alignment[index - 1], alignment[index]), + (Some(previous), Some(next)) if previous + 1 == next + ); + let structural = [&authored[index - 1], &authored[index]] + .iter() + .any(|token| STRUCTURAL_PUNCTUATION.contains(&token.raw.as_str())); + let call = + is_unquoted_word(&authored[index - 1].raw) && authored[index].raw == OPEN_PARENTHESIS; + let inside_parentheses = + authored[index - 1].raw == OPEN_PARENTHESIS || authored[index].raw == CLOSE_PARENTHESIS; + if (adjacent_in_source && !oracle_pair && (!structural || call)) + || (inside_parentheses && !separators[index].contains('\n')) + { + separators[index].clear(); + } else if separators[index].is_empty() + && !adjacent_in_source + && would_fuse(&authored[index - 1], &authored[index]) + { + separators[index] = " ".to_string(); + } + } + Ok(separators) +} + +fn merged_gap(gaps: &[String]) -> String { + if let [only] = gaps { + return only.clone(); + } + if let Some(newline) = gaps.iter().rev().find(|gap| gap.contains('\n')) { + return newline.clone(); + } + if gaps.iter().any(|gap| !gap.is_empty()) { + " ".to_string() + } else { + String::new() + } +} + +fn default_space(previous: &Keyed, next: &Keyed) -> &'static str { + if NO_SPACE_BEFORE.contains(&next.raw.as_str()) + || NO_SPACE_AFTER.contains(&previous.raw.as_str()) + { + "" + } else { + " " + } +} + +/// Whether printing two separated tokens without a space could lex them differently. +fn would_fuse(previous: &Keyed, next: &Keyed) -> bool { + let (Some(left), Some(right)) = (previous.raw.chars().last(), next.raw.chars().next()) else { + return false; + }; + let word = |character: char| { + character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') + }; + let operator = |character: char| "+-*/<>=!|&^%~:#?".contains(character); + (word(left) && word(right)) || (operator(left) && operator(right)) +} + +/// Whether the guide prints the aligned authored word as an upper-case keyword. +fn confirms_keyword(authored: &Keyed, generated: &Keyed) -> bool { + authored.recasable + && generated.keyword + && generated.raw != authored.raw + && generated.raw == generated.raw.to_ascii_uppercase() + && generated.raw.eq_ignore_ascii_case(&authored.raw) +} + +fn attach_comments<'a>(lexemes: &[Lexeme], comments: &'a [Comment]) -> Vec> { + let mut attached: Vec> = (0..lexemes.len()).map(|_| Attached::default()).collect(); + for comment in comments { + let next = lexemes.partition_point(|lexeme| lexeme.start < comment.end); + let preceding = lexemes.partition_point(|lexeme| lexeme.end <= comment.start); + let next = (next < lexemes.len()).then_some(next); + let previous = preceding.checked_sub(1); + match (comment.leading, next, previous) { + (true, Some(next), _) | (false, Some(next), None) => { + attached[next].leading.push(comment); + } + (_, _, Some(previous)) => attached[previous].trailing.push(comment), + (_, None, None) => {} + } + } + attached +} + +fn current_indentation(output: &str) -> String { + let line = output.rsplit('\n').next().unwrap_or_default(); + line.chars() + .take_while(|character| *character == ' ' || *character == '\t') + .collect() +} diff --git a/crates/sqlquack-format/src/format/_helpers/touching.rs b/crates/sqlquack-format/src/format/_helpers/touching.rs new file mode 100644 index 000000000..b9fa830c4 --- /dev/null +++ b/crates/sqlquack-format/src/format/_helpers/touching.rs @@ -0,0 +1,102 @@ +//! The spacing guard: whitespace changes between tokens are checked on the bytes, not the tokenizer. + +use crate::format::_helpers::lexemes::{Lexeme, is_keyword}; +use crate::format::_helpers::token_layout::AuthoredSql; +use crate::format::constants::{ + GUARD_OPERATORS, JOIN_FAILURE, OPERATOR_BYTES, SEPARABLE_KEYWORDS, SEPARABLE_OPERATORS, + SEPARATION_FAILURE, UNSEPARABLE_NEIGHBOURS, +}; + +/// Refuse output that separates touching tokens, or joins separated ones, where that could change their reading. +pub(crate) fn verify_touching_tokens( + before: &AuthoredSql<'_>, + after: &AuthoredSql<'_>, + binary_operators: &[bool], +) -> Result<(), String> { + let pairs = before.lexemes.windows(2).zip(after.lexemes.windows(2)); + for (index, (source, printed)) in pairs.enumerate() { + let touched = source[0].end == source[1].start; + let touches = printed[0].end == printed[1].start; + let operator = |offset: usize| { + binary_operators[index + offset] + && separable_operator(before.sql, &source[offset], &source[1 - offset]) + }; + if touched && !touches && !separable(&source[0], &source[1]) && !operator(0) && !operator(1) + { + return Err(SEPARATION_FAILURE.to_string()); + } + if !touched && touches && could_fuse(&source[0].raw, &source[1].raw) { + return Err(JOIN_FAILURE.to_string()); + } + } + Ok(()) +} + +/// Whether whitespace may go between touching tokens: after a comma, just inside a bracket, around a parenthesis, or beside a clause keyword. +fn separable(previous: &Lexeme, next: &Lexeme) -> bool { + let clause = |lexeme: &Lexeme| { + is_keyword(lexeme) + && SEPARABLE_KEYWORDS + .iter() + .any(|keyword| lexeme.raw.eq_ignore_ascii_case(keyword)) + }; + let numeric = [previous, next] + .iter() + .any(|lexeme| lexeme.raw.starts_with(|first: char| first.is_ascii_digit())); + matches!(previous.raw.as_str(), "," | ";" | "(" | ")" | "[" | "{") + || matches!(next.raw.as_str(), "(" | ")" | "]" | "}") + || (!numeric && (clause(previous) || clause(next))) +} + +/// Whether the parse tree's binary `operator` may be spaced from `neighbour`: the guard's own lexing of the source finds exactly it, and neither side is a path or name accessor. +fn separable_operator(sql: &str, operator: &Lexeme, neighbour: &Lexeme) -> bool { + let bytes = sql.as_bytes(); + let is_operator_byte = |byte: &u8| OPERATOR_BYTES.as_bytes().contains(byte); + let start = bytes[..operator.start] + .iter() + .rposition(|byte| !is_operator_byte(byte)) + .map_or(0, |at| at + 1); + let end = bytes[operator.end..] + .iter() + .position(|byte| !is_operator_byte(byte)) + .map_or(sql.len(), |at| operator.end + at); + let exponent = matches!(operator.raw.as_str(), "+" | "-") + && start == operator.start + && matches!(&bytes[..start], [.., digit, b'e' | b'E'] if digit.is_ascii_digit() || *digit == b'.'); + let accessor = UNSEPARABLE_NEIGHBOURS.contains(&neighbour.raw.as_str()) + || neighbour.raw.starts_with(['@', '$']); + SEPARABLE_OPERATORS.contains(&operator.raw.as_str()) + && !exponent + && !accessor + && !sql[start..end].contains(':') + && operator_pieces(&sql[start..end], start).contains(&(operator.start, operator.end)) +} + +/// The byte ranges of a run of operator characters, read left to right taking the longest operator each time. +fn operator_pieces(run: &str, offset: usize) -> Vec<(usize, usize)> { + let mut pieces: Vec<(usize, usize)> = Vec::new(); + let mut at = 0; + while at < run.len() { + let width = GUARD_OPERATORS + .iter() + .filter(|operator| run[at..].starts_with(*operator)) + .map(|operator| operator.len()) + .max() + .unwrap_or(1); + pieces.push((offset + at, offset + at + width)); + at += width; + } + pieces +} + +/// Whether printing `previous` and `next` with no space between them could read as different tokens. +fn could_fuse(previous: &str, next: &str) -> bool { + let (Some(left), Some(right)) = (previous.chars().last(), next.chars().next()) else { + return false; + }; + let word = |character: char| { + character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') + }; + let operator = |character: char| OPERATOR_BYTES.contains(character); + (word(left) && word(right)) || (operator(left) && operator(right)) +} diff --git a/crates/sqlquack-format/src/format/constants.rs b/crates/sqlquack-format/src/format/constants.rs new file mode 100644 index 000000000..6da828444 --- /dev/null +++ b/crates/sqlquack-format/src/format/constants.rs @@ -0,0 +1,320 @@ +//! Constants of format. + +/// Spaces per indent step when the caller does not choose. +pub const DEFAULT_INDENT: usize = 2; + +pub(crate) const TOKENIZE_FAILURE: &str = "formatter tokenization failed"; +pub(crate) const PARSE_FAILURE: &str = "formatter could not parse SQL"; +pub(crate) const NOT_IDEMPOTENT_FAILURE: &str = "formatter output is not idempotent"; +pub(crate) const TOKEN_PRESERVATION_FAILURE: &str = + "formatter would change authored SQL tokens, not only layout"; +pub(crate) const COMMENT_ATTACHMENT_FAILURE: &str = + "formatter could not preserve comment token attachments"; +pub(crate) const STRING_BREAK_FAILURE: &str = + "formatter would join adjacent string literals across a line break"; +pub(crate) const ALIGNMENT_FAILURE: &str = + "formatter could not align its layout with authored tokens"; +pub(crate) const SEPARATION_FAILURE: &str = + "formatter would separate tokens written touching where the space could change their meaning"; +pub(crate) const JOIN_FAILURE: &str = + "formatter would join tokens written apart where they could read as one"; +pub(crate) const SELECT_LIST_FAILURE: &str = + "formatter could not find every select list, whose items without an alias it keeps as written"; +pub(crate) const UNSUPPORTED_DIALECT_FAILURE: &str = + "formatter does not support this dialect yet; only Snowflake has a format gate"; +pub(crate) const DEPTH_FAILURE: &str = + "formatter does not lay out SQL whose parse tree is more than 512 levels deep"; +/// The deepest parse tree the formatter walks; a deeper one, such as a chain of thousands of `+`, is refused before it can exhaust the stack. +pub(crate) const MAX_TREE_DEPTH: usize = 512; +pub(crate) const TYPE_DEPTH_FAILURE: &str = + "formatter does not lay out data types nested more than 64 levels deep"; +/// The deepest data type the formatter lets the parser build: a run of bracket suffixes such as `INT[][]`, nested type constructors, or both alternating. +pub(crate) const MAX_TYPE_NESTING: usize = 64; +/// The type names whose parameters are named fields, such as `STRUCT(a INT)`. +pub(crate) const STRUCT_TYPE_WORDS: [&str; 5] = ["STRUCT", "ROW", "OBJECT", "UNION", "RECORD"]; +/// The type names whose parameters are data types in angle brackets or parentheses, such as `ARRAY` or `STRUCT(a INT)`. +pub(crate) const TYPE_CONSTRUCTOR_WORDS: [&str; 10] = [ + "ARRAY", "STRUCT", "MAP", "RANGE", "OBJECT", "ROW", "RECORD", "UNION", "VECTOR", "NULLABLE", +]; +pub(crate) const UNSUPPORTED_SYNTAX_FAILURE: &str = + "formatter does not lay out bind variables or string stage references with options"; + +/// Snowflake calls the guide keeps on one line however wide; others split past the guide width. +pub(crate) const SNOWFLAKE_INLINE_CALLS: [&str; 81] = [ + "ABS", + "AI_AGG", + "AI_SUMMARIZE_AGG", + "APPROX_PERCENTILE_ACCUMULATE", + "APPROX_PERCENTILE_COMBINE", + "APPROX_PERCENTILE_ESTIMATE", + "APPROX_TOP_K", + "APPROX_TOP_K_ACCUMULATE", + "APPROX_TOP_K_COMBINE", + "APPROX_TOP_K_ESTIMATE", + "ARRAY_APPEND", + "ARRAY_CONTAINS", + "ARRAY_DISTINCT", + "ARRAY_PREPEND", + "ARRAY_REMOVE", + "ASIN", + "ATAN", + "ATAN2", + "BITMAP_CONSTRUCT_AGG", + "BIT_AND", + "BIT_OR", + "BIT_XOR", + "BOOLXOR_AGG", + "CAST", + "CEIL", + "CEILING", + "COALESCE", + "COS", + "COUNT", + "CURRENT_SCHEMAS", + "DAY", + "DAYOFMONTH", + "DAYOFWEEK", + "DAYOFYEAR", + "DEGREES", + "EXP", + "FLATTEN", + "FLOOR", + "GENERATOR", + "HASH_AGG", + "HLL", + "HOUR", + "KURTOSIS", + "LOG", + "MAP_CONTAINS_KEY", + "MAP_KEYS", + "MAX", + "MIN", + "MINUTE", + "MODE", + "MONTH", + "MONTHS_BETWEEN", + "NEXT_DAY", + "NTH_VALUE", + "OBJECT_AGG", + "PARSE_JSON", + "PERCENTILE_CONT", + "PERCENTILE_DISC", + "POW", + "QUARTER", + "RADIANS", + "RANDOM", + "REGR_AVGX", + "REGR_AVGY", + "REGR_COUNT", + "REGR_INTERCEPT", + "REGR_R2", + "REGR_SLOPE", + "REGR_SXX", + "REGR_SXY", + "REGR_SYY", + "SECOND", + "STDDEV", + "STDDEV_POP", + "STDDEV_SAMP", + "SUM", + "TABLE", + "TAN", + "TRIM", + "TRY_CAST", + "YEAR", +]; +/// DuckDB calls the guide keeps on one line however wide. +pub(crate) const DUCKDB_INLINE_CALLS: [&str; 25] = [ + "AVG", + "CAST", + "COALESCE", + "COUNT", + "CUME_DIST", + "EPOCH_MS", + "EXP", + "FIRST", + "JSON_EXTRACT", + "LAST_VALUE", + "LEN", + "LENGTH", + "LOG", + "LOGICAL_OR", + "MIN", + "MODE", + "NTH_VALUE", + "NTILE", + "PERCENTILE_CONT", + "PERCENTILE_DISC", + "PERCENT_RANK", + "RANK", + "SUM", + "TRIM", + "TRY_CAST", +]; +/// PostgreSQL calls the guide keeps on one line however wide. +pub(crate) const POSTGRES_INLINE_CALLS: [&str; 14] = [ + "ARRAY_LENGTH", + "CAST", + "COUNT", + "DATE_PART", + "EXP", + "EXTRACT", + "JSON_AGG", + "LOGICAL_OR", + "MODE", + "PERCENTILE_CONT", + "SUM", + "TRIM", + "TRY_CAST", + "XMLELEMENT", +]; + +/// The line width past which the layout guide splits a call's arguments or a CASE. +pub(crate) const GUIDE_WIDTH: usize = 80; +/// Spaces per indent step of the layout guide; printing rescales it to the requested indent. +pub(crate) const GUIDE_INDENT: usize = 2; +/// The measured width of a literal, path key or type name before its sequence number's digits. +pub(crate) const PLACEHOLDER_WIDTH: usize = 20; +/// The added width of a `::` cast, which the guide measures as `CAST(value AS type)`. +pub(crate) const POSTFIX_CAST_WIDTH: usize = 10; +/// Keywords the guide always separates from a following `(`. +pub(crate) const SPACED_BEFORE_PARENTHESIS: [&str; 42] = [ + "ALL", + "AND", + "ANY", + "AS", + "AT", + "BEFORE", + "BERNOULLI", + "BLOCK", + "BY", + "CHANGES", + "CUBE", + "DISTINCT", + "ELSE", + "END", + "EXISTS", + "FILTER", + "FROM", + "GROUP", + "HAVING", + "IN", + "JOIN", + "LATERAL", + "MATCH_CONDITION", + "NOT", + "NULL", + "ON", + "OR", + "OVER", + "QUALIFY", + "REPEATABLE", + "ROLLUP", + "SAMPLE", + "SEED", + "SELECT", + "SOME", + "SYSTEM", + "TABLESAMPLE", + "THEN", + "USING", + "VALUES", + "WHEN", + "WHERE", +]; +/// Words that may precede JOIN in a join keyword. +pub(crate) const JOIN_PREFIXES: [&str; 12] = [ + "NATURAL", + "ASOF", + "LEFT", + "RIGHT", + "FULL", + "INNER", + "OUTER", + "CROSS", + "SEMI", + "ANTI", + "POSITIONAL", + "DIRECTED", +]; +/// Typed literal keywords whose string the guide measures with it as one placeholder. +pub(crate) const TYPED_LITERALS: [&str; 7] = [ + "DATE", + "TIME", + "TIMESTAMP", + "TIMESTAMPTZ", + "TIMESTAMP_NTZ", + "TIMESTAMP_LTZ", + "TIMESTAMP_TZ", +]; +/// Words that continue a `::` cast type, such as `DOUBLE PRECISION`. +pub(crate) const TYPE_CONTINUATIONS: [&str; 6] = + ["PRECISION", "VARYING", "WITH", "WITHOUT", "TIME", "ZONE"]; +/// Calls that always put each argument on its own line. +pub(crate) const GROUPING_CALLS: [&str; 2] = ["ROLLUP", "CUBE"]; +/// Clause words of a Snowflake SEMANTIC_VIEW call, each starting its own line. +pub(crate) const SEMANTIC_VIEW_CLAUSES: [&str; 4] = ["METRICS", "DIMENSIONS", "FACTS", "WHERE"]; +/// Clause keywords that start a line of a SELECT. +pub(crate) const CLAUSE_KEYWORDS: [&str; 8] = [ + "FROM", "WHERE", "HAVING", "QUALIFY", "WINDOW", "LIMIT", "OFFSET", "FETCH", +]; +/// Keywords that whitespace may separate from a token they touch, such as `)FROM`. +pub(crate) const SEPARABLE_KEYWORDS: [&str; 14] = [ + "SELECT", + "FROM", + "WHERE", + "GROUP", + "HAVING", + "QUALIFY", + "WINDOW", + "ORDER", + "LIMIT", + "OFFSET", + "FETCH", + "UNION", + "EXCEPT", + "INTERSECT", +]; +/// Binary operator nodes of the parse tree, with the spellings of their operator token. +pub(crate) const BINARY_OPERATOR_NODES: [(&str, &[&str]); 10] = [ + ("eq", &["="]), + ("neq", &["!=", "<>"]), + ("lt", &["<"]), + ("lte", &["<="]), + ("gt", &[">"]), + ("gte", &[">="]), + ("add", &["+"]), + ("sub", &["-"]), + ("mul", &["*"]), + ("concat", &["||"]), +]; +/// The bytes the spacing guard reads as operator characters. +pub(crate) const OPERATOR_BYTES: &str = "+-*/<>=!|&^%~:#?"; +/// Operators the spacing guard lexes, longest match first, so a piece is never part of a longer one. +pub(crate) const GUARD_OPERATORS: [&str; 22] = [ + "->>", "<=>", "||", "<=", ">=", "<>", "!=", "=>", "->", "::", ":=", "/*", "*/", "--", "**", + "//", "<<", ">>", "|>", "!~", "~*", "#>", +]; +/// Binary operators that whitespace may separate from a token they touch. +pub(crate) const SEPARABLE_OPERATORS: [&str; 12] = [ + "=", "<>", "!=", "<", ">", "<=", ">=", "+", "-", "*", "||", "->", +]; +/// Tokens a binary operator is never spaced from: name and path accessors. +pub(crate) const UNSEPARABLE_NEIGHBOURS: [&str; 3] = [".", ":", "::"]; +/// Keywords after which a clause word is a table alias instead. +pub(crate) const ALIAS_FOLLOWERS: [&str; 3] = ["MATCH_CONDITION", "ON", "USING"]; +/// Set operators, each starting its own line between its operands. +pub(crate) const SET_OPERATORS: [&str; 4] = ["UNION", "EXCEPT", "INTERSECT", "MINUS"]; +/// Words that continue a set operator, such as `UNION ALL BY NAME`. +pub(crate) const SET_OPERATOR_MODIFIERS: [&str; 5] = + ["ALL", "DISTINCT", "BY", "NAME", "CORRESPONDING"]; +/// The branch keywords of a CASE expression. +pub(crate) const CASE_BRANCHES: [&str; 3] = ["WHEN", "THEN", "ELSE"]; +/// The sampling unit spaced before `(` after SAMPLE or TABLESAMPLE. +pub(crate) const SAMPLE_ROW_KEYWORD: &str = "ROW"; +/// The Snowflake call whose clauses start their own lines. +pub(crate) const SEMANTIC_VIEW_CALL: &str = "SEMANTIC_VIEW"; +/// An opening parenthesis. +pub(crate) const OPEN_PARENTHESIS: &str = "("; +/// A closing parenthesis. +pub(crate) const CLOSE_PARENTHESIS: &str = ")"; diff --git a/crates/sqlquack-format/src/format/main/format_sql.rs b/crates/sqlquack-format/src/format/main/format_sql.rs new file mode 100644 index 000000000..21b6564ce --- /dev/null +++ b/crates/sqlquack-format/src/format/main/format_sql.rs @@ -0,0 +1,9 @@ +//! `format_sql`: SQL laid out over lines, with exactly its tokens apart from keyword case. + +use crate::format::_helpers::pipeline::format_checked; +use crate::format::models::{FormatOptions, FormatRefusal, Formatted}; + +/// `sql` laid out over lines; refused, with a reason, when that cannot be done safely. +pub fn format_sql(sql: &str, options: &FormatOptions) -> Result { + format_checked(sql, options).map_err(|reason| FormatRefusal { reason }) +} diff --git a/crates/sqlquack-format/src/format/main/mod.rs b/crates/sqlquack-format/src/format/main/mod.rs new file mode 100644 index 000000000..0cd7ebcc7 --- /dev/null +++ b/crates/sqlquack-format/src/format/main/mod.rs @@ -0,0 +1,3 @@ +//! Entry points of format. + +pub mod format_sql; diff --git a/crates/sqlquack-format/src/format/mod.rs b/crates/sqlquack-format/src/format/mod.rs new file mode 100644 index 000000000..63cbaa265 --- /dev/null +++ b/crates/sqlquack-format/src/format/mod.rs @@ -0,0 +1,9 @@ +//! Format: SQL laid out over lines, changing only the whitespace between tokens and keyword case. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-format/src/format/models.rs b/crates/sqlquack-format/src/format/models.rs new file mode 100644 index 000000000..f24a1bcbb --- /dev/null +++ b/crates/sqlquack-format/src/format/models.rs @@ -0,0 +1,51 @@ +//! Data models of format. + +use std::collections::HashMap; + +use sqlquack_syntax::{Dialect, ParseOptions}; + +use crate::format::constants::DEFAULT_INDENT; + +/// How SQL is laid out. +#[derive(Debug, Clone)] +pub struct FormatOptions { + /// The dialect the SQL is tokenized and parsed in. + pub dialect: Dialect, + /// Lines longer than this are re-broken; absent, only the parse-tree layout applies. + pub line_width: Option, + /// Spaces per indent step. + pub indent: usize, + /// Printed widths of tokens that stand in for longer text, keyed by upper-case token text. + pub token_widths: HashMap, + /// Complexity guards for the validating parse. + pub parse: ParseOptions, +} + +impl FormatOptions { + /// Default layout for `dialect`: no line width, two-space indent. + pub fn new(dialect: Dialect) -> Self { + Self { + dialect, + line_width: None, + indent: DEFAULT_INDENT, + token_widths: HashMap::new(), + parse: ParseOptions::default(), + } + } +} + +/// Formatted SQL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Formatted { + /// The laid-out SQL; the input when nothing changed. + pub sql: String, + /// Whether `sql` differs from the input. + pub changed: bool, +} + +/// Why SQL was left as it was written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FormatRefusal { + /// The reason, for display. + pub reason: String, +} diff --git a/crates/sqlquack-format/src/format/tests.rs b/crates/sqlquack-format/src/format/tests.rs new file mode 100644 index 000000000..3dc57e4d0 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_format_corpus.rs"] +mod test_format_corpus; +#[path = "tests/test_format_sql.rs"] +mod test_format_sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-format/src/format/tests/fixtures/FORMAT_CORPUS_NOTICE.md b/crates/sqlquack-format/src/format/tests/fixtures/FORMAT_CORPUS_NOTICE.md new file mode 100644 index 000000000..39e0868f7 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/fixtures/FORMAT_CORPUS_NOTICE.md @@ -0,0 +1,32 @@ +# Formatter corpus notice + +`format_corpus.json` contains SELECT and WITH queries taken from the SQLGlot dialect identity test +fixtures for BigQuery, Databricks, DuckDB, PostgreSQL, Snowflake and T-SQL, as extracted by the +Polyglot project. One literal was reworded. The expected `formatted` and `previously_formatted` +values are SQLBuild formatter output. + +The queries are used under the following license: + +```text +MIT License + +Copyright (c) 2026 Toby Mao + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/crates/sqlquack-format/src/format/tests/fixtures/format_corpus.json b/crates/sqlquack-format/src/format/tests/fixtures/format_corpus.json new file mode 100644 index 000000000..b45dd71a3 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/fixtures/format_corpus.json @@ -0,0 +1,5354 @@ +[ + { + "dialect": "bigquery", + "sql": "SELECT 'foo' 'bar'", + "formatted": "SELECT 'foo' 'bar'" + }, + { + "dialect": "bigquery", + "sql": "SELECT 'foo'/* c */'bar'", + "formatted": "SELECT\n 'foo' /* c */ 'bar'" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.mydataset.mytable`", + "formatted": "SELECT *\nFROM `domain.com:project-id.mydataset.mytable`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS", + "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA`.JOBS" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", + "formatted": "SELECT *\nFROM `domain.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `a.b.com:project-id.mydataset.mytable`", + "formatted": "SELECT *\nFROM `a.b.com:project-id.mydataset.mytable`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`", + "formatted": "SELECT *\nFROM `a.b.com:project-id.region-us.INFORMATION_SCHEMA.JOBS`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x-0.y", + "formatted": "SELECT *\nFROM x-0.y" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.UdF`(data) FROM `p.d.t`", + "formatted": "SELECT `p.d.UdF`(data)\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY_CONCAT([1])", + "formatted": "SELECT ARRAY_CONCAT([1])" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM READ_CSV('bla.csv')", + "formatted": "SELECT *\nFROM READ_CSV('bla.csv')" + }, + { + "dialect": "bigquery", + "sql": "SELECT jsondoc['some_key']", + "formatted": "SELECT jsondoc['some_key']" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.UdF`(data).* FROM `p.d.t`", + "formatted": "SELECT `p.d.UdF`(data).*\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `my-project.my-dataset.my-table`", + "formatted": "SELECT *\nFROM `my-project.my-dataset.my-table`" + }, + { + "dialect": "bigquery", + "sql": "SELECT x, 1 AS y GROUP BY 1 ORDER BY 1", + "formatted": "SELECT\n x,\n 1 AS y\nGROUP BY\n 1\nORDER BY\n 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.*", + "formatted": "SELECT *\nFROM x.*" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x.y*", + "formatted": "SELECT *\nFROM x.y*" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_KEYS_AND_VALUES')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')", + "formatted": "SELECT SEARCH(data_to_search, 'search_query', analyzer_options => 'analyzer_options_values')" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', json_scope => 'JSON_VALUES', analyzer => 'LOG_ANALYZER')", + "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n json_scope => 'JSON_VALUES',\n analyzer => 'LOG_ANALYZER'\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT SEARCH(data_to_search, 'search_query', analyzer => 'PATTERN_ANALYZER', analyzer_options => 'options')", + "formatted": "SELECT\n SEARCH(\n data_to_search,\n 'search_query',\n analyzer => 'PATTERN_ANALYZER',\n analyzer_options => 'options'\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)", + "formatted": "SELECT *\nFROM dataset.my_table TABLESAMPLE SYSTEM (10 PERCENT)" + }, + { + "dialect": "bigquery", + "sql": "SELECT '\\n\\r\\a\\v\\f\\t'", + "formatted": "SELECT '\\n\\r\\a\\v\\f\\t'" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM tbl FOR SYSTEM_TIME AS OF z", + "formatted": "SELECT *\nFROM tbl FOR SYSTEM_TIME AS OF z" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM tbl FOR SYSTEM TIME AS OF z", + "formatted": "SELECT *\nFROM tbl FOR SYSTEM TIME AS OF z" + }, + { + "dialect": "bigquery", + "sql": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')", + "formatted": "SELECT PARSE_TIMESTAMP('%c', 'Thu Dec 25 07:30:00 2008', 'UTC')" + }, + { + "dialect": "bigquery", + "sql": "SELECT ANY_VALUE(fruit HAVING MAX sold) FROM fruits", + "formatted": "SELECT ANY_VALUE(fruit HAVING MAX sold)\nFROM fruits" + }, + { + "dialect": "bigquery", + "sql": "SELECT ANY_VALUE(fruit HAVING MIN sold) FROM fruits", + "formatted": "SELECT ANY_VALUE(fruit HAVING MIN sold)\nFROM fruits" + }, + { + "dialect": "bigquery", + "sql": "SELECT `project-id`.udfs.func(call.dir)", + "formatted": "SELECT `project-id`.udfs.func(call.dir)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day", + "formatted": "SELECT CAST(CURRENT_DATE AS STRING FORMAT 'DAY') AS current_day" + }, + { + "dialect": "bigquery", + "sql": "SELECT foo IN UNNEST(bar) AS bla", + "formatted": "SELECT foo IN UNNEST(bar) AS bla" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x-0.a", + "formatted": "SELECT *\nFROM x-0.a" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM pivot CROSS JOIN foo", + "formatted": "SELECT *\nFROM pivot\nCROSS JOIN foo" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b-c.mydataset.mytable", + "formatted": "SELECT *\nFROM a-b-c.mydataset.mytable" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM abc-def-ghi", + "formatted": "SELECT *\nFROM abc-def-ghi" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b-c", + "formatted": "SELECT *\nFROM a-b-c" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM my-table", + "formatted": "SELECT *\nFROM my-table" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM my-project.mydataset.mytable", + "formatted": "SELECT *\nFROM my-project.mydataset.mytable" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM pro-ject_id.c.d CROSS JOIN foo-bar", + "formatted": "SELECT *\nFROM pro-ject_id.c.d\nCROSS JOIN foo-bar" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25", + "formatted": "SELECT *\nFROM foo.bar.25" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25_", + "formatted": "SELECT *\nFROM foo.bar.25_" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25x a", + "formatted": "SELECT *\nFROM foo.bar.25x a" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo.bar.25ab c", + "formatted": "SELECT *\nFROM foo.bar.25ab c" + }, + { + "dialect": "bigquery", + "sql": "SELECT b'abc'", + "formatted": "SELECT b'abc'" + }, + { + "dialect": "bigquery", + "sql": "SELECT AS STRUCT 1 AS a, 2 AS b", + "formatted": "SELECT AS STRUCT\n 1 AS a,\n 2 AS b" + }, + { + "dialect": "bigquery", + "sql": "SELECT DISTINCT AS STRUCT 1 AS a, 2 AS b", + "formatted": "SELECT DISTINCT AS STRUCT\n 1 AS a,\n 2 AS b" + }, + { + "dialect": "bigquery", + "sql": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)", + "formatted": "SELECT AS VALUE STRUCT(1 AS a, 2 AS b)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM q UNPIVOT(values FOR quarter IN (b, c))", + "formatted": "SELECT *\nFROM q UNPIVOT(values FOR quarter IN (b, c))" + }, + { + "dialect": "bigquery", + "sql": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)", + "formatted": "SELECT MAKE_INTERVAL(100, 11, 1, 12, 30, 10)" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 FROM x GROUP BY y + 1 ORDER BY 1", + "formatted": "SELECT y + 1\nFROM x\nGROUP BY\n y + 1\nORDER BY\n 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT TIMESTAMP_SECONDS(2) AS t", + "formatted": "SELECT TIMESTAMP_SECONDS(2) AS t" + }, + { + "dialect": "bigquery", + "sql": "SELECT TIMESTAMP_MILLIS(2) AS t", + "formatted": "SELECT TIMESTAMP_MILLIS(2) AS t" + }, + { + "dialect": "bigquery", + "sql": "SELECT COUNT(x RESPECT NULLS)", + "formatted": "SELECT COUNT(x RESPECT NULLS)" + }, + { + "dialect": "bigquery", + "sql": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x", + "formatted": "SELECT LAST_VALUE(x IGNORE NULLS) OVER y AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b))", + "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ))" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY((SELECT AS STRUCT 1 AS a, 2 AS b) LIMIT 10)", + "formatted": "SELECT\n ARRAY((\n SELECT AS STRUCT\n 1 AS a,\n 2 AS b\n ) LIMIT 10)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM x WHERE x.y >= (SELECT MAX(a) FROM b-c) - 20", + "formatted": "SELECT *\nFROM x\nWHERE\n x.y >= (\n SELECT MAX(a)\n FROM b-c\n )\n - 20" + }, + { + "dialect": "bigquery", + "sql": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts", + "formatted": "SELECT FORMAT_TIMESTAMP('%F %T', CURRENT_TIMESTAMP(), 'Europe/Berlin') AS ts" + }, + { + "dialect": "bigquery", + "sql": "SELECT cars, apples FROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))", + "formatted": "SELECT\n cars,\n apples\nFROM some_table PIVOT(SUM(total_counts) FOR products IN ('general.cars' AS cars, 'food.apples' AS apples))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM test QUALIFY a IS DISTINCT FROM b WINDOW c AS (PARTITION BY d)", + "formatted": "SELECT *\nFROM test\nQUALIFY a IS DISTINCT FROM b\nWINDOW\n c AS (PARTITION BY d)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM (SELECT * FROM `t`) AS a UNPIVOT((c) FOR c_name IN (v1, v2))", + "formatted": "SELECT *\nFROM (\n SELECT *\n FROM `t`\n) AS a UNPIVOT((c) FOR c_name IN (v1, v2))" + }, + { + "dialect": "bigquery", + "sql": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING) FROM x WINDOW y AS (PARTITION BY CATEGORY)", + "formatted": "SELECT ROW() OVER (y ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" + }, + { + "dialect": "bigquery", + "sql": "SELECT item, purchases, LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular FROM Produce WINDOW item_window AS (ORDER BY purchases)", + "formatted": "SELECT\n item,\n purchases,\n LAST_VALUE(item) OVER (item_window ROWS BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS most_popular\nFROM Produce\nWINDOW\n item_window AS (ORDER BY purchases)" + }, + { + "dialect": "bigquery", + "sql": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y FROM x WINDOW y AS (PARTITION BY CATEGORY)", + "formatted": "SELECT LAST_VALUE(a IGNORE NULLS) OVER y\nFROM x\nWINDOW\n y AS (PARTITION BY CATEGORY)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'", + "formatted": "SELECT *\nFROM foo AS t0 FOR SYSTEM_TIME AS OF '2026-02-12T23:22:21.743416+00:00'" + }, + { + "dialect": "bigquery", + "sql": "SELECT b\"\\x0a$'x'00\"", + "formatted": "SELECT b\"\\x0a$'x'00\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t1, t2", + "formatted": "SELECT *\nFROM t1, t2" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\\t\"", + "formatted": "SELECT r\"\\t\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`", + "formatted": "SELECT *\nFROM `proj.dataset.INFORMATION_SCHEMA.SOME_VIEW`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES", + "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name", + "formatted": "SELECT *\nFROM region_or_dataset.INFORMATION_SCHEMA.TABLES AS some_name" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES", + "formatted": "SELECT *\nFROM proj.region_or_dataset.INFORMATION_SCHEMA.TABLES" + }, + { + "dialect": "bigquery", + "sql": "SELECT ARRAY(SELECT AS STRUCT 1 a, 2 b)", + "formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 a,\n 2 b)", + "previously_formatted": "SELECT\n ARRAY(SELECT AS STRUCT\n 1 AS a,\n 2 AS b)" + }, + { + "dialect": "bigquery", + "sql": "select array_contains([1, 2, 3], 1)", + "formatted": "SELECT array_contains([1, 2, 3], 1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT SPLIT(foo)", + "formatted": "SELECT SPLIT(foo)" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS hash", + "formatted": "SELECT 1 AS hash" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS at", + "formatted": "SELECT 1 AS at" + }, + { + "dialect": "bigquery", + "sql": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"", + "formatted": "SELECT \"\"\"ends with \\\"word\\\"\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT '''ends with \\'word\\''''", + "formatted": "SELECT '''ends with \\'word\\''''" + }, + { + "dialect": "bigquery", + "sql": "SELECT \"\"\"a\\\"b\"\"\"", + "formatted": "SELECT \"\"\"a\\\"b\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\"\"ends with \\\"\"\"\"", + "formatted": "SELECT r\"\"\"ends with \\\"\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT r\"\"\"a\\\"b\"\"\"", + "formatted": "SELECT r\"\"\"a\\\"b\"\"\"" + }, + { + "dialect": "bigquery", + "sql": "SELECT a overlaps", + "formatted": "SELECT a overlaps", + "previously_formatted": "SELECT\n a AS overlaps" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 z FROM x GROUP BY y + 1 ORDER BY z", + "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z", + "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1\nORDER BY\n z" + }, + { + "dialect": "bigquery", + "sql": "SELECT y + 1 z FROM x GROUP BY y + 1", + "formatted": "SELECT y + 1 z\nFROM x\nGROUP BY\n y + 1", + "previously_formatted": "SELECT\n y + 1 AS z\nFROM x\nGROUP BY\n y + 1" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON '\"foo\"' AS json_data", + "formatted": "SELECT JSON '\"foo\"' AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM (SELECT a, b, c FROM test) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", + "formatted": "SELECT *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) d, COUNT(*) e FOR c IN ('x', 'y'))", + "previously_formatted": "SELECT\n *\nFROM (\n SELECT\n a,\n b,\n c\n FROM test\n) PIVOT(SUM(b) AS d, COUNT(*) AS e FOR c IN ('x', 'y'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(1 AS BYTEINT)", + "formatted": "SELECT CAST(1 AS BYTEINT)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM a-b c", + "formatted": "SELECT *\nFROM a-b c", + "previously_formatted": "SELECT\n *\nFROM a-b AS c" + }, + { + "dialect": "bigquery", + "sql": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')", + "formatted": "SELECT PARSE_DATETIME('%a %b %e %I:%M:%S %Y', 'Thu Dec 25 07:30:00 2008')" + }, + { + "dialect": "bigquery", + "sql": "SELECT TRANSLATE(MODEL, 'in', 't') FROM (SELECT 'input' AS MODEL)", + "formatted": "SELECT TRANSLATE(MODEL, 'in', 't')\nFROM (\n SELECT 'input' AS MODEL\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT GRANT FROM (SELECT 'input' AS GRANT)", + "formatted": "SELECT GRANT\nFROM (\n SELECT 'input' AS GRANT\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(1 AS NUMERIC(10, 2))", + "formatted": "SELECT CAST(1 AS NUMERIC(10, 2))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST('1' AS STRING(10)) UNION ALL SELECT CAST('2' AS STRING(10))", + "formatted": "SELECT CAST('1' AS STRING(10))\nUNION ALL\nSELECT CAST('2' AS STRING(10))" + }, + { + "dialect": "bigquery", + "sql": "SELECT cola FROM (SELECT CAST('1' AS STRING(10)) AS cola UNION ALL SELECT CAST('2' AS STRING(10)) AS cola)", + "formatted": "SELECT cola\nFROM (\n SELECT CAST('1' AS STRING(10)) AS cola\n UNION ALL\n SELECT CAST('2' AS STRING(10)) AS cola\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT a, b, c, d, e FROM GAP_FILL(TABLE foo, ts_column => 'b', partitioning_columns => ['a'], value_columns => [('c', 'bar'), ('d', 'baz'), ('e', 'bla')], bucket_width => INTERVAL '1' DAY)", + "formatted": "SELECT\n a,\n b,\n c,\n d,\n e\nFROM GAP_FILL(\n TABLE foo,\n ts_column => 'b',\n partitioning_columns => ['a'],\n value_columns => [\n ('c', 'bar'),\n ('d', 'baz'),\n ('e', 'bla')\n ],\n bucket_width => INTERVAL '1' DAY\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'linear')], ignore_null_values => FALSE) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'linear')],\n ignore_null_values => FALSE\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'null')], origin => CAST('2023-11-01 09:30:01' AS DATETIME)) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'null')],\n origin => CAST('2023-11-01 09:30:01' AS DATETIME)\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM GAP_FILL(TABLE device_data, ts_column => 'time', bucket_width => INTERVAL '1' MINUTE, value_columns => [('signal', 'locf')]) ORDER BY time", + "formatted": "SELECT *\nFROM GAP_FILL(\n TABLE device_data,\n ts_column => 'time',\n bucket_width => INTERVAL '1' MINUTE,\n value_columns => [('signal', 'locf')]\n)\nORDER BY\n time" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT label, column1, column2 FROM mydataset.mytable))", + "formatted": "SELECT *\nFROM ML.PREDICT(MODEL mydataset.mymodel, (\n SELECT\n label,\n column1,\n column2\n FROM mydataset.mytable\n))" + }, + { + "dialect": "bigquery", + "sql": "SELECT label, predicted_label1, predicted_label AS predicted_label2 FROM ML.PREDICT(MODEL mydataset.mymodel2, (SELECT * EXCEPT (predicted_label), predicted_label AS predicted_label1 FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)))", + "formatted": "SELECT\n label,\n predicted_label1,\n predicted_label AS predicted_label2\nFROM ML.PREDICT(\n MODEL mydataset.mymodel2,\n (\n SELECT\n * EXCEPT (predicted_label),\n predicted_label AS predicted_label1\n FROM ML.PREDICT(MODEL mydataset.mymodel1, TABLE mydataset.mytable)\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL mydataset.mymodel, (SELECT custom_label, column1, column2 FROM mydataset.mytable), STRUCT(0.55 AS threshold))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL mydataset.mymodel,\n (\n SELECT\n custom_label,\n column1,\n column2\n FROM mydataset.mytable\n ),\n STRUCT(0.55 AS threshold)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (SELECT * FROM input_data))", + "formatted": "SELECT *\nFROM ML.PREDICT(MODEL `my_project`.my_dataset.my_model, (\n SELECT *\n FROM input_data\n))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input FROM my_dataset.object_table))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 480, 480, FALSE) AS input\n FROM my_dataset.object_table\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.PREDICT(MODEL my_dataset.vision_model, (SELECT uri, ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input FROM my_dataset.object_table WHERE content_type = 'image/jpeg'))", + "formatted": "SELECT *\nFROM ML.PREDICT(\n MODEL my_dataset.vision_model,\n (\n SELECT\n uri,\n ML.CONVERT_COLOR_SPACE(ML.RESIZE_IMAGE(ML.DECODE_IMAGE(data), 224, 280, TRUE), 'YIQ') AS input\n FROM my_dataset.object_table\n WHERE content_type = 'image/jpeg'\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FEATURES_AT_TIME((SELECT 1), num_rows => 1)", + "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME((\n SELECT 1\n), num_rows => 1)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FEATURES_AT_TIME(TABLE mydataset.feature_table, time => '2022-06-11 10:00:00+00', num_rows => 1, ignore_feature_nulls => TRUE)", + "formatted": "SELECT *\nFROM ML.FEATURES_AT_TIME(\n TABLE mydataset.feature_table,\n time => '2022-06-11 10:00:00+00',\n num_rows => 1,\n ignore_feature_nulls => TRUE\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table, query_column_to_search => 'query_column_to_search', top_k => 2, distance_type => 'cosine', options => '{\"fraction_lists_to_search\":0.15}')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n TABLE mydataset.base_table,\n 'column_to_search',\n TABLE mydataset.query_table,\n query_column_to_search => 'query_column_to_search',\n top_k => 2,\n distance_type => 'cosine',\n options => '{\"fraction_lists_to_search\":0.15}'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH((SELECT * FROM mydataset.base_table), 'column_to_search', (SELECT * FROM mydataset.query_table), 'query_column_to_search')", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(\n (\n SELECT *\n FROM mydataset.base_table\n ),\n 'column_to_search',\n (\n SELECT *\n FROM mydataset.query_table\n ),\n 'query_column_to_search'\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)", + "formatted": "SELECT *\nFROM VECTOR_SEARCH(TABLE mydataset.base_table, 'column_to_search', TABLE mydataset.query_table)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mytranslatemodel`, TABLE `mydataset.mybqtable`, STRUCT('translate_text' AS translate_mode, 'zh-CN' AS target_language_code))", + "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mytranslatemodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(\n 'translate_text' AS translate_mode,\n 'zh-CN' AS target_language_code\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))", + "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, STRUCT(2 AS horizon))" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, TABLE `mydataset.mybqtable`, STRUCT(2 AS horizon, 4 AS confidence_level))", + "formatted": "SELECT *\nFROM ML.FORECAST(\n MODEL `mydataset.mymodel`,\n TABLE `mydataset.mybqtable`,\n STRUCT(2 AS horizon, 4 AS confidence_level)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.FORECAST(MODEL `mydataset.mymodel`, (SELECT * FROM mydataset.query_table), STRUCT())", + "formatted": "SELECT *\nFROM ML.FORECAST(MODEL `mydataset.mymodel`, (\n SELECT *\n FROM mydataset.query_table\n), STRUCT())" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.FORECAST(TABLE citibike_trips, data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", + "formatted": "SELECT *\nFROM AI.FORECAST(\n TABLE citibike_trips,\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.FORECAST((SELECT * FROM citibike_trips), data_col => 'num_trips', timestamp_col => 'date', horizon => 30)", + "formatted": "SELECT *\nFROM AI.FORECAST(\n (\n SELECT *\n FROM citibike_trips\n ),\n data_col => 'num_trips',\n timestamp_col => 'date',\n horizon => 30\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", + "formatted": "SELECT *\nFROM ML.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.GENERATE_TEXT(MODEL `mydataset.gemini_model`, TABLE `mydataset.prompt_table`, STRUCT(0.15 AS temperature))", + "formatted": "SELECT *\nFROM AI.GENERATE_TEXT(\n MODEL `mydataset.gemini_model`,\n TABLE `mydataset.prompt_table`,\n STRUCT(0.15 AS temperature)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM AI.GENERATE_TABLE(MODEL `mydataset.gemini_model`, (SELECT 'Q' AS prompt), STRUCT('name STRING' AS output_schema))", + "formatted": "SELECT *\nFROM AI.GENERATE_TABLE(\n MODEL `mydataset.gemini_model`,\n (\n SELECT 'Q' AS prompt\n ),\n STRUCT('name STRING' AS output_schema)\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')", + "formatted": "SELECT AI.GENERATE_BOOL(MODEL `mydataset.gemini_model`, 'Is sky blue?')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.EMBED('hello')", + "formatted": "SELECT AI.EMBED('hello')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.SIMILARITY('a', 'b')", + "formatted": "SELECT AI.SIMILARITY('a', 'b')" + }, + { + "dialect": "bigquery", + "sql": "SELECT AI.GENERATE('Write a haiku')", + "formatted": "SELECT AI.GENERATE('Write a haiku')" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT() AS json_data", + "formatted": "SELECT JSON_OBJECT() AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data", + "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', TRUE) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data", + "formatted": "SELECT JSON_OBJECT('foo', 10, 'bar', ['a', 'b']) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data", + "formatted": "SELECT JSON_OBJECT('a', 10, 'a', 'foo') AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [10, NULL]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [JSON '10', JSON '\"foo\"']) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [STRUCT(10 AS id, 'Red' AS color), STRUCT(20 AS id, 'Blue' AS color)]) AS json_data", + "formatted": "SELECT\n JSON_OBJECT(['a', 'b'], [\n STRUCT(10 AS id, 'Red' AS color),\n STRUCT(20 AS id, 'Blue' AS color)\n ]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data", + "formatted": "SELECT JSON_OBJECT(['a', 'b'], [TO_JSON(10), TO_JSON(['foo', 'bar'])]) AS json_data" + }, + { + "dialect": "bigquery", + "sql": "SELECT MOD((SELECT 1), 2)", + "formatted": "SELECT\n MOD((\n SELECT 1\n ), 2)" + }, + { + "dialect": "bigquery", + "sql": "SELECT STRUCT>([\"2023-01-17\"])", + "formatted": "SELECT STRUCT> ([\"2023-01-17\"])" + }, + { + "dialect": "bigquery", + "sql": "SELECT STRUCT((SELECT 'foo')).*", + "formatted": "SELECT\n STRUCT ((\n SELECT 'foo'\n )) .*" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-12-01' AS DATE), CAST('2022-12-31' AS DATE))", + "formatted": "SELECT\n RANGE(\n CAST('2022-12-01' AS DATE),\n CAST('2022-12-31' AS DATE)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))", + "formatted": "SELECT RANGE(NULL, CAST('2022-12-31' AS DATE))" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27' AS DATETIME), CAST('2022-10-01 16:00:00' AS DATETIME))", + "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27' AS DATETIME),\n CAST('2022-10-01 16:00:00' AS DATETIME)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT RANGE(CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP), CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP))", + "formatted": "SELECT\n RANGE(\n CAST('2022-10-01 14:53:27 America/Los_Angeles' AS TIMESTAMP),\n CAST('2022-10-01 16:00:00 America/Los_Angeles' AS TIMESTAMP)\n )" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST('2026-03-24' AS STRING FORMAT ('YYYY'))", + "formatted": "SELECT CAST('2026-03-24' AS STRING FORMAT('YYYY'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY')) FROM (SELECT DATE('2026-03-24') AS date)", + "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY'))\nFROM (\n SELECT DATE('2026-03-24') AS date\n)" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS STRING FORMAT ('YYYY-MM-DD'))", + "formatted": "SELECT CAST(date AS STRING FORMAT('YYYY-MM-DD'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(timestamp AS STRING FORMAT ('YYYY-MM-DD') AT TIME ZONE 'UTC')", + "formatted": "SELECT CAST(timestamp AS STRING FORMAT('YYYY-MM-DD') AT TIME ZONE 'UTC')" + }, + { + "dialect": "bigquery", + "sql": "SELECT CAST(date AS TIMESTAMP FORMAT ('YYYY-MM-DD HH24:MI:SS'))", + "formatted": "SELECT CAST(date AS TIMESTAMP FORMAT('YYYY-MM-DD HH24:MI:SS'))" + }, + { + "dialect": "bigquery", + "sql": "SELECT a, GROUP_CONCAT(b) FROM table GROUP BY a", + "formatted": "SELECT\n a,\n GROUP_CONCAT(b)\nFROM table\nGROUP BY\n a" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS foo INNER UNION ALL SELECT 3 AS foo, 4 AS bar", + "formatted": "SELECT 1 AS foo\nINNER UNION ALL\nSELECT\n 3 AS foo,\n 4 AS bar" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL CORRESPONDING BY (foo, bar) SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x LEFT UNION ALL CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nLEFT UNION ALL CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT 1 AS x UNION ALL STRICT CORRESPONDING BY (foo, bar) SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL STRICT CORRESPONDING BY (foo, bar)\nSELECT 2 AS x" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM UNNEST(x) WITH OFFSET EXCEPT DISTINCT SELECT * FROM UNNEST(y) WITH OFFSET", + "formatted": "SELECT *\nFROM UNNEST(x) WITH OFFSET\nEXCEPT DISTINCT\nSELECT *\nFROM UNNEST(y) WITH OFFSET" + }, + { + "dialect": "bigquery", + "sql": "SELECT col FROM t WHERE _PARTITIONTIME BETWEEN a AND b", + "formatted": "SELECT col\nFROM t\nWHERE _PARTITIONTIME BETWEEN a AND b" + }, + { + "dialect": "bigquery", + "sql": "SELECT _DBT_MAX_PARTITION FROM t", + "formatted": "SELECT _DBT_MAX_PARTITION\nFROM t" + }, + { + "dialect": "bigquery", + "sql": "WITH cte(c) AS (SELECT * FROM t) SELECT * FROM cte", + "formatted": "WITH cte(c) AS (\n SELECT *\n FROM t\n)\n\nSELECT *\nFROM cte" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t AS t(c1, c2)", + "formatted": "SELECT *\nFROM t AS t(c1, c2)" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM t AS t(c1, c2)", + "formatted": "SELECT *\nFROM t AS t(c1, c2)" + }, + { + "dialect": "bigquery", + "sql": "WITH foo AS (SELECT [1, 2, 3] AS array_col) SELECT array_col[offset] FROM foo CROSS JOIN UNNEST(array_col) WITH OFFSET AS offset", + "formatted": "WITH foo AS (\n SELECT [1, 2, 3] AS array_col\n)\n\nSELECT array_col[offset]\nFROM foo\nCROSS JOIN UNNEST(array_col) WITH OFFSET AS offset" + }, + { + "dialect": "bigquery", + "sql": "SELECT `db.t`.`c` FROM `db.t`", + "formatted": "SELECT `db.t`.`c`\nFROM `db.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT `p.d.t`.`c`.`f` FROM `p.d.t`", + "formatted": "SELECT `p.d.t`.`c`.`f`\nFROM `p.d.t`" + }, + { + "dialect": "bigquery", + "sql": "SELECT * FROM ML.TRANSLATE(MODEL `mydataset.mymodel`, (SELECT comment AS text_content FROM mydataset.mytable), STRUCT('translate_text' AS translate_mode, 'en' AS target_language_code))", + "formatted": "SELECT *\nFROM ML.TRANSLATE(\n MODEL `mydataset.mymodel`,\n (\n SELECT comment AS text_content\n FROM mydataset.mytable\n ),\n STRUCT(\n 'translate_text' AS translate_mode,\n 'en' AS target_language_code\n )\n)" + }, + { + "dialect": "bigquery", + "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT JSON_EXTRACT(c, '$.x-y') FROM t", + "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT JSON_EXTRACT(c, '$.x-y')\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)", + "formatted": "SELECT CAST('a' AS STRING COLLATE UTF8_BINARY)" + }, + { + "dialect": "databricks", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_BIT_POSITION(10)", + "formatted": "SELECT BITMAP_BIT_POSITION(10)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", + "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" + }, + { + "dialect": "databricks", + "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", + "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" + }, + { + "dialect": "databricks", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "databricks", + "sql": "SELECT MODE(category)", + "formatted": "SELECT MODE(category)" + }, + { + "dialect": "databricks", + "sql": "SELECT MODE(price, TRUE) AS deterministic_mode FROM products", + "formatted": "SELECT MODE(price, TRUE) AS deterministic_mode\nFROM products" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST(NULL AS VOID)", + "formatted": "SELECT CAST(NULL AS VOID)" + }, + { + "dialect": "databricks", + "sql": "SELECT void FROM t", + "formatted": "SELECT void\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM stream", + "formatted": "SELECT *\nFROM stream" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM STREAM t", + "formatted": "SELECT *\nFROM STREAM t" + }, + { + "dialect": "databricks", + "sql": "SELECT t.current_time FROM t", + "formatted": "SELECT t.current_time\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT col1 FROM t CLUSTER BY col1", + "formatted": "SELECT col1\nFROM t\nCLUSTER BY\n col1" + }, + { + "dialect": "databricks", + "sql": "SELECT col1, col2 FROM t CLUSTER BY col1, col2", + "formatted": "SELECT\n col1,\n col2\nFROM t\nCLUSTER BY\n col1,\n col2" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO HOUR)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO MINUTE)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)", + "formatted": "SELECT CAST('11 23:4:0' AS INTERVAL DAY TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO MINUTE)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL HOUR TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)", + "formatted": "SELECT CAST('23:00:00' AS INTERVAL MINUTE TO SECOND)" + }, + { + "dialect": "databricks", + "sql": "SELECT ${x} FROM ${y} WHERE ${z} > 1", + "formatted": "SELECT ${x}\nFROM ${y}\nWHERE ${z} > 1" + }, + { + "dialect": "databricks", + "sql": "SELECT PARSE_JSON('{}')", + "formatted": "SELECT PARSE_JSON('{}')" + }, + { + "dialect": "databricks", + "sql": "SELECT RANDSTR(123)", + "formatted": "SELECT RANDSTR(123)" + }, + { + "dialect": "databricks", + "sql": "SELECT RANDSTR(123, 456)", + "formatted": "SELECT RANDSTR(123, 456)" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", + "formatted": "SELECT *\nFROM sales UNPIVOT INCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))", + "formatted": "SELECT *\nFROM sales UNPIVOT EXCLUDE NULLS (sales FOR quarter IN (q1 AS `Jan-Mar`))" + }, + { + "dialect": "databricks", + "sql": "SELECT * FROM t1, t2", + "formatted": "SELECT *\nFROM t1, t2" + }, + { + "dialect": "databricks", + "sql": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE", + "formatted": "SELECT TIMESTAMP '2025-04-29 18.47.18'::DATE" + }, + { + "dialect": "databricks", + "sql": "SELECT DATE_FORMAT(CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP), 'yyyy-MM-dd HH:mm:ss') AS foo FROM t", + "formatted": "SELECT\n DATE_FORMAT(\n CAST(FROM_UTC_TIMESTAMP(foo, 'America/Los_Angeles') AS TIMESTAMP),\n 'yyyy-MM-dd HH:mm:ss'\n ) AS foo\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT r\"\\\\foo.bar\\\"", + "formatted": "SELECT r\"\\\\foo.bar\\\"" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTRING_INDEX(str, delim, count)", + "formatted": "SELECT SUBSTRING_INDEX(str, delim, count)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR('Spark' FROM 5 FOR 1)", + "formatted": "SELECT SUBSTR('Spark' FROM 5 FOR 1)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR('Spark SQL', 5)", + "formatted": "SELECT SUBSTR('Spark SQL', 5)" + }, + { + "dialect": "databricks", + "sql": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)", + "formatted": "SELECT SUBSTR(ENCODE('Spark SQL', 'utf-8'), 5)" + }, + { + "dialect": "databricks", + "sql": "SELECT test, LISTAGG(email, '') AS Email FROM organizations GROUP BY test", + "formatted": "SELECT\n test,\n LISTAGG(email, '') AS Email\nFROM organizations\nGROUP BY\n test" + }, + { + "dialect": "databricks", + "sql": "WITH t AS (VALUES ('foo_val') AS t(foo1)) SELECT foo1 FROM t", + "formatted": "WITH t AS (\n VALUES ('foo_val') AS t(foo1)\n)\n\nSELECT foo1\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result", + "formatted": "SELECT ELT(2, 'foo', 'bar', 'baz') AS Result" + }, + { + "dialect": "databricks", + "sql": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)", + "formatted": "SELECT MAKE_INTERVAL(100, 11, 12, 13, 14, 14, 15)" + }, + { + "dialect": "databricks", + "sql": "SELECT name, GROUPING_ID() FROM customer GROUP BY ROLLUP (name)", + "formatted": "SELECT\n name,\n GROUPING_ID()\nFROM customer\nGROUP BY\n ROLLUP (\n name\n )" + }, + { + "dialect": "databricks", + "sql": "SELECT CURDATE()", + "formatted": "SELECT CURDATE()" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:price, c1:price.foo, c1:price.bar[1]", + "formatted": "SELECT\n c1:price,\n c1:price.foo,\n c1:price.bar[1]" + }, + { + "dialect": "databricks", + "sql": "SELECT TRY_CAST(c1:price AS ARRAY)", + "formatted": "SELECT TRY_CAST(c1:price AS ARRAY)" + }, + { + "dialect": "databricks", + "sql": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)", + "formatted": "SELECT TRY_CAST(c1:[\"foo bar\"][\"baz qux\"] AS ARRAY)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:item[1].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT c1:item[1].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:item[*].price FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT c1:item[*].price\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0] FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT FROM_JSON(c1:item[*].price, 'ARRAY')[0]\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>')) FROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)", + "formatted": "SELECT INLINE(FROM_JSON(c1:item[*], 'ARRAY>'))\nFROM VALUES ('{ \"item\": [ { \"model\" : \"basic\", \"price\" : 6.12 }, { \"model\" : \"medium\", \"price\" : 9.24 } ] }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT c1:['price'] FROM VALUES ('{ \"price\": 5 }') AS T(c1)", + "formatted": "SELECT c1:['price']\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(c1, '$.price') FROM VALUES ('{ \"price\": 5 }') AS T(c1)", + "formatted": "SELECT GET_JSON_OBJECT(c1, '$.price')\nFROM VALUES ('{ \"price\": 5 }') AS T(c1)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(col, path_col)", + "formatted": "SELECT GET_JSON_OBJECT(col, path_col)" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))", + "formatted": "SELECT GET_JSON_OBJECT(col, CONCAT('$.', field_name))" + }, + { + "dialect": "databricks", + "sql": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')", + "formatted": "SELECT GET_JSON_OBJECT(GET_JSON_OBJECT(col, '$[0]'), '$.a')" + }, + { + "dialect": "databricks", + "sql": "SELECT raw:`zip code`, raw:`fb:testid`, raw:store['bicycle'], raw:store[\"zip code\"]", + "formatted": "SELECT\n raw:`zip code`,\n raw:`fb:testid`,\n raw:store['bicycle'],\n raw:store[\"zip code\"]" + }, + { + "dialect": "databricks", + "sql": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')", + "formatted": "SELECT DATE_ADD(MONTH, 1, '2020-01-01')" + }, + { + "dialect": "databricks", + "sql": "SELECT DATEADD(e, 24) FROM t", + "formatted": "SELECT DATEADD(e, 24)\nFROM t" + }, + { + "dialect": "databricks", + "sql": "SELECT '20'?::INTEGER", + "formatted": "SELECT '20'?::INTEGER" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)", + "formatted": "SELECT OVERLAY('Spark SQL', 'ANSI ', 7, 0)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)", + "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'CORE' FROM 7)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)", + "formatted": "SELECT OVERLAY(ENCODE('Spark SQL', 'utf-8') PLACING ENCODE('_', 'utf-8') FROM 6)" + }, + { + "dialect": "databricks", + "sql": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)", + "formatted": "SELECT OVERLAY('Spark SQL' PLACING 'ANSI ' FROM 7 FOR 0)" + }, + { + "dialect": "databricks", + "sql": "WITH t AS (SELECT '{\"x-y\": \"z\"}' AS c) SELECT get_json_object(c, '$.x-y') FROM t", + "formatted": "WITH t AS (\n SELECT '{\"x-y\": \"z\"}' AS c\n)\n\nSELECT get_json_object(c, '$.x-y')\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT ([1,2,3])[:-:-1]", + "formatted": "SELECT\n (\n [1, 2, 3]\n )[:-:-1]" + }, + { + "dialect": "duckdb", + "sql": "SELECT INTERVAL '1 hour'::VARCHAR", + "formatted": "SELECT INTERVAL '1 hour'::VARCHAR" + }, + { + "dialect": "duckdb", + "sql": "SELECT str[0:1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MODE(category)", + "formatted": "SELECT MODE(category)" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'\\n'", + "formatted": "SELECT e'\\n'" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'\\t'", + "formatted": "SELECT e'\\t'" + }, + { + "dialect": "duckdb", + "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", + "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" + }, + { + "dialect": "duckdb", + "sql": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)", + "formatted": "SELECT GET_BIT(CAST('0110010' AS BIT), 2)" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 WHERE x > $1", + "formatted": "SELECT 1\nWHERE x > $1" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 WHERE x > $name", + "formatted": "SELECT 1\nWHERE x > $name" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 AS a$b", + "formatted": "SELECT 1 AS a$b" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"x\": 1}' -> c FROM t", + "formatted": "SELECT '{\"x\": 1}' -> c\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'", + "formatted": "SELECT '{\"duck\": [1, 2, 3]}' -> '$.duck[#-1]'" + }, + { + "dialect": "duckdb", + "sql": "SELECT tbl.x*1e4+tbl.y FROM tbl", + "formatted": "SELECT tbl.x * 1e4 + tbl.y\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)", + "formatted": "SELECT LIST_TRANSFORM([5, NULL, 6], (x, y) -> COALESCE(x, y, 0) + 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y : COALESCE(x, y, 0) + 1)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x : x % 2 = 0), LAMBDA y : y * y)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x : COALESCE(x, 0) + 1)", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_TRANSFORM(nbr, LAMBDA x : x + 1) FROM article AS a", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM my_ducklake.demo AT (VERSION => 2)", + "formatted": "SELECT *\nFROM my_ducklake.demo AT (VERSION => 2)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t1 AS a AT (VERSION => 3) JOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK) ON a.id = b.id", + "formatted": "SELECT *\nFROM t1 AS a AT (VERSION => 3)\nJOIN t2 AS b AT (TIMESTAMP => NOW() - INTERVAL '1' WEEK)\n ON a.id = b.id" + }, + { + "dialect": "duckdb", + "sql": "SELECT TO_BINARY('test')", + "formatted": "SELECT TO_BINARY('test')" + }, + { + "dialect": "duckdb", + "sql": "SELECT UUIDV7()", + "formatted": "SELECT UUIDV7()" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRY(LOG(0))", + "formatted": "SELECT TRY(LOG(0))" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1 AS x UNION ALL BY NAME SELECT 2 AS x", + "formatted": "SELECT 1 AS x\nUNION ALL BY NAME\nSELECT 2 AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT SUM(x) FILTER (x = 1)", + "formatted": "SELECT SUM(x) FILTER(x = 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM GLOB(x)", + "formatted": "SELECT *\nFROM GLOB(x)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAP(['key1', 'key2', 'key3'], [10, 20, 30])", + "formatted": "SELECT\n MAP(\n ['key1', 'key2', 'key3'],\n [10, 20, 30]\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAP {'x': 1}", + "formatted": "SELECT MAP {'x': 1}" + }, + { + "dialect": "duckdb", + "sql": "SELECT (MAP {'x': 1})['x']", + "formatted": "SELECT\n (\n MAP {'x': 1}\n )['x']" + }, + { + "dialect": "duckdb", + "sql": "SELECT df1.*, df2.* FROM df1 POSITIONAL JOIN df2", + "formatted": "SELECT\n df1.*,\n df2.*\nFROM df1\nPOSITIONAL JOIN df2" + }, + { + "dialect": "duckdb", + "sql": "SELECT EPOCH_MS(10) AS t", + "formatted": "SELECT EPOCH_MS(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAKE_TIMESTAMP(10) AS t", + "formatted": "SELECT MAKE_TIMESTAMP(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT TO_TIMESTAMP(10) AS t", + "formatted": "SELECT TO_TIMESTAMP(10) AS t" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST(col, recursive := TRUE) FROM t", + "formatted": "SELECT UNNEST(col, recursive := TRUE)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo ASOF LEFT JOIN bar ON a = b", + "formatted": "SELECT *\nFROM foo\nASOF LEFT JOIN bar\n ON a = b" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'a': 1} AS x", + "formatted": "SELECT {'a': 1} AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x", + "formatted": "SELECT {'a': {'b': {'c': 1}}, 'd': {'e': 2}} AS x" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'x': 1, 'y': 2, 'z': 3}", + "formatted": "SELECT {'x': 1, 'y': 2, 'z': 3}" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}", + "formatted": "SELECT {'key1': 'string', 'key2': 1, 'key3': 12.345}" + }, + { + "dialect": "duckdb", + "sql": "SELECT ROW(x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", + "formatted": "SELECT ROW(x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT (x, x + 1, y) FROM (SELECT 1 AS x, 'a' AS y)", + "formatted": "SELECT (x, x + 1, y)\nFROM (\n SELECT\n 1 AS x,\n 'a' AS y\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT a.x FROM (SELECT {'x': 1, 'y': 2, 'z': 3} AS a)", + "formatted": "SELECT a.x\nFROM (\n SELECT {'x': 1, 'y': 2, 'z': 3} AS a\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t1 WHERE NOT EXISTS(FROM t2 WHERE t2.id = t1.id)", + "formatted": "SELECT *\nFROM t1\nWHERE\n NOT EXISTS(\n FROM t2\n WHERE t2.id = t1.id\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo WHERE bar > $baz AND bla = $bob", + "formatted": "SELECT *\nFROM foo\nWHERE\n bar > $baz AND bla = $bob" + }, + { + "dialect": "duckdb", + "sql": "SELECT STAR(tbl, exclude := [foo])", + "formatted": "SELECT STAR(tbl, exclude := [foo])" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY GROUPING SETS (species), GROUPING SETS (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n GROUPING SETS (\n species\n ),\n GROUPING SETS (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY CUBE (species), CUBE (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n CUBE (\n species\n ),\n CUBE (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT species, island, COUNT(*) FROM t GROUP BY ROLLUP (species), ROLLUP (island)", + "formatted": "SELECT\n species,\n island,\n COUNT(*)\nFROM t\nGROUP BY\n ROLLUP (\n species\n ),\n ROLLUP (\n island\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']", + "formatted": "SELECT '{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }' ->> ['$.family', '$.species']" + }, + { + "dialect": "duckdb", + "sql": "SELECT $🦆$foo$🦆$", + "formatted": "SELECT $🦆$foo$🦆$" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))", + "formatted": "SELECT *\nFROM t PIVOT(FIRST(t) AS t, FOR quarter IN ('Q1', 'Q2'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])", + "formatted": "SELECT JSON_EXTRACT_STRING('{ \"family\": \"anatidae\", \"species\": [ \"duck\", \"goose\", \"swan\", null ] }', ['$.family', '$.species'])" + }, + { + "dialect": "duckdb", + "sql": "SELECT col FROM t WHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')", + "formatted": "SELECT col\nFROM t\nWHERE JSON_EXTRACT_STRING(col, '$.id') NOT IN ('b')" + }, + { + "dialect": "duckdb", + "sql": "SELECT a, LOGICAL_OR(b) FROM foo GROUP BY a", + "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM foo\nGROUP BY\n a" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'", + "formatted": "SELECT JSON_EXTRACT_STRING(c, '$.k1') = 'v1'" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'", + "formatted": "SELECT JSON_EXTRACT(c, '$.k1') = 'v1'" + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0", + "formatted": "SELECT '{\"foo\": [1, 2, 3]}' -> 'foo' -> 0" + }, + { + "dialect": "duckdb", + "sql": "SELECT ($$hello)'world$$)", + "formatted": "SELECT\n (\n $$hello)'world$$\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT $$foo$$", + "formatted": "SELECT $$foo$$" + }, + { + "dialect": "duckdb", + "sql": "SELECT $tag$foo$tag$", + "formatted": "SELECT $tag$foo$tag$" + }, + { + "dialect": "duckdb", + "sql": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}", + "formatted": "SELECT {'yes': 'duck', 'maybe': 'goose', 'huh': NULL, 'no': 'heron'}" + }, + { + "dialect": "duckdb", + "sql": "SELECT a['x space'] FROM (SELECT {'x space': 1, 'y': 2, 'z': 3} AS a)", + "formatted": "SELECT a['x space']\nFROM (\n SELECT {'x space': 1, 'y': 2, 'z': 3} AS a\n)" + }, + { + "dialect": "duckdb", + "sql": "WITH pivot_alias AS (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) SELECT * FROM pivot_alias", + "formatted": "WITH pivot_alias AS (\n PIVOT Cities ON Year USING SUM(Population) GROUP BY Country\n)\n\nSELECT *\nFROM pivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias", + "formatted": "SELECT *\nFROM (PIVOT Cities ON Year USING SUM(Population) GROUP BY Country) AS pivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)", + "formatted": "SELECT *\nFROM cities PIVOT(SUM(population) FOR year IN (2000, 2010, 2020) GROUP BY country)" + }, + { + "dialect": "duckdb", + "sql": "SELECT schema_name, function_name, ROW_NUMBER() OVER my_window AS function_rank FROM DUCKDB_FUNCTIONS() WINDOW my_window AS (PARTITION BY schema_name ORDER BY function_name) QUALIFY ROW_NUMBER() OVER my_window < 3", + "formatted": "SELECT\n schema_name,\n function_name,\n ROW_NUMBER() OVER my_window AS function_rank\nFROM DUCKDB_FUNCTIONS()\nWINDOW\n my_window AS (PARTITION BY schema_name ORDER BY function_name)\nQUALIFY ROW_NUMBER() OVER my_window < 3" + }, + { + "dialect": "duckdb", + "sql": "SELECT SHA256('abc')", + "formatted": "SELECT SHA256('abc')" + }, + { + "dialect": "duckdb", + "sql": "SELECT SPLIT_PART('11.22.33', '.', 1)", + "formatted": "SELECT SPLIT_PART('11.22.33', '.', 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", + "formatted": "SELECT NTH_VALUE(is_deleted, 2) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted FROM my_table", + "formatted": "SELECT NTH_VALUE(is_deleted, 2 IGNORE NULLS) OVER (PARTITION BY id) AS nth_is_deleted\nFROM my_table" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_MAX(values) FROM table1", + "formatted": "SELECT LIST_MAX(values)\nFROM table1" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_MIN(values) FROM table1", + "formatted": "SELECT LIST_MIN(values)\nFROM table1" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC')", + "formatted": "SELECT LIST_SORT(x, 'ASC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC')", + "formatted": "SELECT LIST_SORT(x, 'DESC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')", + "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')", + "formatted": "SELECT LIST_SORT(x, 'ASC', 'NULLS LAST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS LAST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DE' || 'SC')", + "formatted": "SELECT LIST_SORT(x, 'DE' || 'SC')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')", + "formatted": "SELECT LIST_SORT(x, 'DESC', 'NULLS' || ' FIRST')" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_SORT(x, 'DE' || 'SC', 'NULLS' || ' FIRST')", + "formatted": "SELECT\n LIST_SORT(\n x,\n 'DE' || 'SC',\n 'NULLS' || ' FIRST'\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", + "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC) FROM t", + "formatted": "SELECT PERCENTILE_DISC(0.25) WITHIN GROUP (ORDER BY y DESC)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 0, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')", + "formatted": "SELECT REGEXP_EXTRACT(a, 'pattern', 1, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1)" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0, 'i')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 1, 'im')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]", + "formatted": "SELECT REGEXP_EXTRACT_ALL(s, 'pattern', 0)[2:]" + }, + { + "dialect": "duckdb", + "sql": "SELECT ISNAN(x)", + "formatted": "SELECT ISNAN(x)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM RANGE(1, 5, 10)", + "formatted": "SELECT *\nFROM RANGE(1, 5, 10)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM GENERATE_SERIES(2, 13, 4)", + "formatted": "SELECT *\nFROM GENERATE_SERIES(2, 13, 4)" + }, + { + "dialect": "duckdb", + "sql": "SELECT i FROM RANGE(5) AS _(i) ORDER BY i ASC", + "formatted": "SELECT i\nFROM RANGE(5) AS _(i)\nORDER BY\n i ASC" + }, + { + "dialect": "duckdb", + "sql": "SELECT i FROM GENERATE_SERIES(12) AS _(i) ORDER BY i ASC", + "formatted": "SELECT i\nFROM GENERATE_SERIES(12) AS _(i)\nORDER BY\n i ASC" + }, + { + "dialect": "duckdb", + "sql": "SELECT LENGTH(foo)", + "formatted": "SELECT LENGTH(foo)" + }, + { + "dialect": "duckdb", + "sql": "SELECT ARRAY[1, 2, 3]", + "formatted": "SELECT ARRAY[1, 2, 3]" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (DESCRIBE t)", + "formatted": "SELECT *\nFROM (DESCRIBE t)" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name", + "formatted": "SELECT UNNEST([*COLUMNS('alias_.*')]) AS column_name" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(*COLUMNS(*)) FROM (SELECT NULL, 2, 3) AS t(a, b, c)", + "formatted": "SELECT COALESCE(*COLUMNS(*))\nFROM (\n SELECT\n NULL,\n 2,\n 3\n) AS t(a, b, c)" + }, + { + "dialect": "duckdb", + "sql": "SELECT id, STRUCT_PACK(*COLUMNS('m\\d')) AS measurements FROM many_measurements", + "formatted": "SELECT\n id,\n STRUCT_PACK(*COLUMNS('m\\d')) AS measurements\nFROM many_measurements" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS(c -> c LIKE '%num%') FROM numbers", + "formatted": "SELECT COLUMNS(c -> c LIKE '%num%')\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT MIN(COLUMNS(* REPLACE (number + id AS number))), COUNT(COLUMNS(* EXCLUDE (number))) FROM numbers", + "formatted": "SELECT\n MIN(COLUMNS(* REPLACE (number + id AS number))),\n COUNT(COLUMNS(* EXCLUDE (number)))\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS(*) + COLUMNS(*) FROM numbers", + "formatted": "SELECT COLUMNS(*) + COLUMNS(*)\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COLUMNS('(id|numbers?)') FROM numbers", + "formatted": "SELECT COLUMNS('(id|numbers?)')\nFROM numbers" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", + "formatted": "SELECT COALESCE(COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result FROM (SELECT NULL AS a, 42 AS b, TRUE AS c)", + "formatted": "SELECT COALESCE(*COLUMNS(['a', 'b', 'c'])) AS result\nFROM (\n SELECT\n NULL AS a,\n 42 AS b,\n TRUE AS c\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)", + "formatted": "SELECT DATE_ADD(CAST('2020-01-01' AS DATE), INTERVAL 1 DAY)" + }, + { + "dialect": "duckdb", + "sql": "SELECT #2, #1 FROM (VALUES (1, 'foo'))", + "formatted": "SELECT\n #2,\n #1\nFROM (VALUES (1, 'foo'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT #2 AS a, #1 AS b FROM (VALUES (1, 'foo'))", + "formatted": "SELECT\n #2 AS a,\n #1 AS b\nFROM (VALUES (1, 'foo'))" + }, + { + "dialect": "duckdb", + "sql": "SELECT CUME_DIST( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT CUME_DIST(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT NTILE(1 ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT PERCENT_RANK( ORDER BY foo) OVER (ORDER BY 1) FROM (SELECT 1 AS foo)", + "formatted": "SELECT PERCENT_RANK(ORDER BY foo) OVER (ORDER BY 1)\nFROM (\n SELECT 1 AS foo\n)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10 PERCENT", + "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10%", + "formatted": "SELECT *\nFROM t\nLIMIT 10%", + "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10 PERCENT OFFSET 1", + "formatted": "SELECT *\nFROM t\nLIMIT 10 PERCENT\nOFFSET 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t LIMIT 10% OFFSET 1", + "formatted": "SELECT *\nFROM t\nLIMIT 10%\nOFFSET 1", + "previously_formatted": "SELECT\n *\nFROM t\nLIMIT 10 %\nOFFSET 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))", + "formatted": "SELECT CAST(ROW(1, 2) AS ROW(a INTEGER, b INTEGER))" + }, + { + "dialect": "duckdb", + "sql": "SELECT row", + "formatted": "SELECT row" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')", + "formatted": "SELECT TRY_STRPTIME('2013-04-28T20:57:01.123456789+07:00', '%Y-%m-%dT%H:%M:%S.%n%z')" + }, + { + "dialect": "duckdb", + "sql": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result", + "formatted": "SELECT REPLACE('apple pie', 'pie', 'cobbler') AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT REPLACE(CAST(CAST('apple pie' AS BLOB) AS TEXT), CAST(CAST('pie' AS BLOB) AS TEXT), CAST(CAST('cobbler' AS BLOB) AS TEXT)) AS result", + "formatted": "SELECT\n REPLACE(\n CAST(CAST('apple pie' AS BLOB) AS TEXT),\n CAST(CAST('pie' AS BLOB) AS TEXT),\n CAST(CAST('cobbler' AS BLOB) AS TEXT)\n ) AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT TRIM('***apple***', '*') AS result", + "formatted": "SELECT TRIM('***apple***', '*') AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result", + "formatted": "SELECT CAST(TRIM(CAST(CAST('***apple***' AS BLOB) AS TEXT), CAST(CAST('*' AS BLOB) AS TEXT)) AS BLOB) AS result" + }, + { + "dialect": "duckdb", + "sql": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)", + "formatted": "SELECT GREATEST(1.0, 2.5, NULL, 3.7)" + }, + { + "dialect": "duckdb", + "sql": "SELECT MAKE_DATE(DATE_PART(['year', 'month', 'day'], CURRENT_DATE))", + "formatted": "SELECT\n MAKE_DATE(\n DATE_PART(\n ['year', 'month', 'day'],\n CURRENT_DATE\n )\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM t PIVOT(SUM(y) FOR foo IN y_enum)", + "formatted": "SELECT *\nFROM t PIVOT(SUM(y) FOR foo IN y_enum)" + }, + { + "dialect": "duckdb", + "sql": "SELECT 20_000 AS literal", + "formatted": "SELECT 20_000 AS literal" + }, + { + "dialect": "duckdb", + "sql": "SELECT 1_2E+1_0::FLOAT", + "formatted": "SELECT 1_2E+1_0::FLOAT" + }, + { + "dialect": "duckdb", + "sql": "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT TODAY()", + "formatted": "SELECT TODAY()" + }, + { + "dialect": "duckdb", + "sql": "SELECT GET_CURRENT_TIME()", + "formatted": "SELECT GET_CURRENT_TIME()" + }, + { + "dialect": "duckdb", + "sql": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t", + "formatted": "SELECT SUM(x) OVER (ORDER BY x GROUPS BETWEEN 1 PRECEDING AND CURRENT ROW)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x) FROM t", + "formatted": "SELECT LAST_VALUE(x ORDER BY x IGNORE NULLS) OVER (ORDER BY x)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x) FROM t", + "formatted": "SELECT LAST_VALUE(x ORDER BY x RESPECT NULLS) OVER (ORDER BY x)\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT CURRENT_DATE", + "formatted": "SELECT CURRENT_DATE" + }, + { + "dialect": "duckdb", + "sql": "SELECT CURRENT_TIMESTAMP", + "formatted": "SELECT CURRENT_TIMESTAMP" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 5", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 5" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10%", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10%" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10 PERCENT (bernoulli)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10 PERCENT (bernoulli)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE reservoir(50 ROWS) REPEATABLE (100)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE reservoir (50 ROWS) REPEATABLE (100)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl USING SAMPLE 10% (system, 377)", + "formatted": "SELECT *\nFROM tbl\nUSING SAMPLE 10% (system, 377)" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl TABLESAMPLE RESERVOIR(20%), tbl2 WHERE tbl.i=tbl2.i", + "formatted": "SELECT *\nFROM tbl TABLESAMPLE RESERVOIR (20%), tbl2\nWHERE tbl.i = tbl2.i", + "previously_formatted": "SELECT\n *\nFROM tbl TABLESAMPLE RESERVOIR (20 %), tbl2\nWHERE\n tbl.i = tbl2.i" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM tbl, tbl2 WHERE tbl.i=tbl2.i USING SAMPLE RESERVOIR(20%)", + "formatted": "SELECT *\nFROM tbl, tbl2\nWHERE tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20%)", + "previously_formatted": "SELECT\n *\nFROM tbl, tbl2\nWHERE\n tbl.i = tbl2.i\nUSING SAMPLE RESERVOIR (20 %)" + }, + { + "dialect": "duckdb", + "sql": "SELECT x::INT[3][3]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT ARRAY[[[1]]]::INT[1][1][1]", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "WITH unpivot_alias AS (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) SELECT * FROM unpivot_alias", + "formatted": "WITH unpivot_alias AS (\n UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales\n)\n\nSELECT *\nFROM unpivot_alias" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias", + "formatted": "SELECT *\nFROM (UNPIVOT monthly_sales ON COLUMNS(* EXCLUDE (empid, dept)) INTO NAME month VALUE sales) AS unpivot_alias" + }, + { + "dialect": "duckdb", + "sql": "WITH cities(country, name, year, population) AS (SELECT 'NL', 'Amsterdam', 2000, 1005 UNION ALL SELECT 'US', 'Seattle', 2020, 738) PIVOT cities ON year USING SUM(population)", + "formatted": "WITH cities(country, name, year, population) AS (\n SELECT\n 'NL',\n 'Amsterdam',\n 2000,\n 1005\n UNION ALL\n SELECT\n 'US',\n 'Seattle',\n 2020,\n 738\n)\n\nPIVOT cities ON year USING SUM(population)" + }, + { + "dialect": "duckdb", + "sql": "WITH t1 AS (FROM (FROM t2 SELECT foo1, foo2)) FROM t1", + "formatted": "WITH t1 AS (\n FROM (FROM t2\n SELECT\n foo1,\n foo2\n )\n)\n\nFROM t1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: 1", + "formatted": "SELECT foo:1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: bar", + "formatted": "SELECT foo:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: t.col FROM t", + "formatted": "SELECT foo:t.col\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\" /* bla */: 1", + "formatted": "SELECT\n \"foo\" /* bla */:1" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": 1 /* bla */", + "formatted": "SELECT \"foo\":1 /* bla */" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": /* bla */ 1", + "formatted": "SELECT\n \"foo\": /* bla */1" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": /* bla */ 1 /* foo */", + "formatted": "SELECT\n \"foo\": /* bla */1 /* foo */" + }, + { + "dialect": "duckdb", + "sql": "SELECT \"foo\": 1", + "formatted": "SELECT \"foo\":1" + }, + { + "dialect": "duckdb", + "sql": "SELECT foo: 1, bar: 2, baz: 3", + "formatted": "SELECT\n foo:1,\n bar:2,\n baz:3" + }, + { + "dialect": "duckdb", + "sql": "SELECT e: 1 + 2, f: len('asdf'), s: (SELECT 42)", + "formatted": "SELECT\n e:1 + 2,\n f:len('asdf'), s:\n (\n SELECT 42\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo: bar", + "formatted": "SELECT *\nFROM foo:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo: c.db.tbl", + "formatted": "SELECT *\nFROM foo:c.db.tbl" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: bar", + "formatted": "SELECT *\nFROM foo /* bla */:bar" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: bar /* baz */", + "formatted": "SELECT *\nFROM foo /* bla */:bar /* baz */" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM foo /* bla */: /* baz */ bar /* boo */", + "formatted": "SELECT *\nFROM foo /* bla */: /* baz */bar /* boo */" + }, + { + "dialect": "duckdb", + "sql": "SELECT * FROM r: range(10), v: (VALUES (42)), s: (FROM range(10))", + "formatted": "SELECT *\nFROM r:range(10), v:(VALUES (42)), s:(\n FROM RANGE(10)\n)" + }, + { + "dialect": "duckdb", + "sql": "\n SELECT\n l_returnflag,\n l_linestatus,\n sum_qty: sum(l_quantity),\n sum_base_price: sum(l_extendedprice),\n sum_disc_price: sum(l_extendedprice * (1-l_discount)),\n sum_charge: sum(l_extendedprice * (1-l_discount) * (1+l_tax)),\n avg_qty: avg(l_quantity),\n avg_price: avg(l_extendedprice),\n avg_disc: avg(l_discount),\n count_order: count(*)\n ", + "formatted": "SELECT\n l_returnflag,\n l_linestatus,\n sum_qty:sum(l_quantity),\n sum_base_price:sum(l_extendedprice),\n sum_disc_price:sum(l_extendedprice\n * (\n 1 - l_discount\n )),\n sum_charge:sum(l_extendedprice\n * (\n 1 - l_discount\n )\n * (\n 1 + l_tax\n )),\n avg_qty:avg(l_quantity),\n avg_price:avg(l_extendedprice),\n avg_disc:avg(l_discount),\n count_order:count(*)" + }, + { + "dialect": "duckdb", + "sql": "SELECT @col FROM t", + "formatted": "SELECT @col\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT @col + 1 FROM t", + "formatted": "SELECT @col + 1\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT (@col) + 1 FROM t", + "formatted": "SELECT\n (@col\n )\n + 1\nFROM t" + }, + { + "dialect": "duckdb", + "sql": "SELECT @(-1)", + "formatted": "SELECT\n @(\n -1\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT @(-1) + 1", + "formatted": "SELECT\n @(\n -1\n )\n + 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT (@-1) + 1", + "formatted": "SELECT\n (@-1\n )\n + 1" + }, + { + "dialect": "duckdb", + "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", + "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (SELECT a, b FROM (VALUES (1, 3), (2, 4)) AS t(a, b) UNION SELECT a + 1, b FROM tbl WHERE a < 3) SELECT * FROM tbl", + "formatted": "WITH RECURSIVE tbl(a, b) USING KEY (a, b) AS (\n SELECT\n a,\n b\n FROM (VALUES (1, 3), (2, 4)) AS t(a, b)\n UNION\n SELECT\n a + 1,\n b\n FROM tbl\n WHERE a < 3\n)\nSELECT *\nFROM tbl" + }, + { + "dialect": "duckdb", + "sql": "WITH _data AS (SELECT [{'a': 1, 'b': 2}, {'a': 2, 'b': 3}] AS col) SELECT t.col['b'] FROM _data, UNNEST(_data.col) AS t(col) WHERE t.col['a'] = 1", + "formatted": "WITH _data AS (\n SELECT\n [\n {'a': 1, 'b': 2},\n {'a': 2, 'b': 3}\n ] AS col\n)\n\nSELECT t.col['b']\nFROM _data, UNNEST(_data.col) AS t(col)\nWHERE t.col['a'] = 1" + }, + { + "dialect": "duckdb", + "sql": "SELECT [4, 5, 6] AS l, [x FOR x, i IN l IF i = 2] AS filtered", + "formatted": "SELECT\n [4, 5, 6] AS l,\n [x FOR x, i IN l IF i = 2] AS filtered" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_VALUE(1)[i]", + "formatted": "SELECT LIST_VALUE(1)[i]" + }, + { + "dialect": "duckdb", + "sql": "SELECT LIST_APPLY(RANGE(1, 4), i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]})", + "formatted": "SELECT\n LIST_APPLY(\n RANGE(1, 4),\n i -> {'f1': LIST_VALUE(1, 2, 3)[i], 'f2': LIST_VALUE(1, 2, 3)[i]}\n )" + }, + { + "dialect": "duckdb", + "sql": "SELECT UNNEST([1, 2])", + "formatted": "SELECT UNNEST([1, 2])" + }, + { + "dialect": "duckdb", + "sql": "SELECT file[:256] FROM GLOB('*')", + "formatted": null + }, + { + "dialect": "duckdb", + "sql": "SELECT file[256] FROM GLOB('*')", + "formatted": "SELECT file[256]\nFROM GLOB('*')" + }, + { + "dialect": "postgres", + "sql": "WITH t AS (SELECT 1 AS \"null\") SELECT t.null FROM t", + "formatted": "WITH t AS (\n SELECT 1 AS \"null\"\n)\n\nSELECT t.null\nFROM t" + }, + { + "dialect": "postgres", + "sql": "WITH t AS (SELECT 1 AS \"true\") SELECT t.true FROM t", + "formatted": "WITH t AS (\n SELECT 1 AS \"true\"\n)\n\nSELECT t.true\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'", + "formatted": "SELECT '%' SIMILAR TO '^%' ESCAPE '^'" + }, + { + "dialect": "postgres", + "sql": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)", + "formatted": "SELECT GET_BIT(CAST(44 AS BIT(10)), 6)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t GROUP BY ROLLUP (a || '^' || b)", + "formatted": "SELECT *\nFROM t\nGROUP BY\n ROLLUP (\n a || '^' || b\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "postgres", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "postgres", + "sql": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common FROM orders", + "formatted": "SELECT MODE() WITHIN GROUP (ORDER BY status DESC) AS most_common\nFROM orders" + }, + { + "dialect": "postgres", + "sql": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist", + "formatted": "SELECT ST_DISTANCE(gg1, gg2, FALSE) AS sphere_dist" + }, + { + "dialect": "postgres", + "sql": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))", + "formatted": "SELECT EXTRACT(QUARTER FROM CAST('2025-04-26' AS DATE))" + }, + { + "dialect": "postgres", + "sql": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))", + "formatted": "SELECT DATE_TRUNC('QUARTER', CAST('2025-04-26' AS DATE))" + }, + { + "dialect": "postgres", + "sql": "SELECT x FROM t WHERE CAST($1 AS TEXT) = 'ok'", + "formatted": "SELECT x\nFROM t\nWHERE CAST($1 AS TEXT) = 'ok'" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)", + "formatted": "SELECT *\nFROM t TABLESAMPLE SYSTEM (50) REPEATABLE (55)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM events WHERE doc @? '$.a[*] ? (@ > 2)'", + "formatted": "SELECT *\nFROM events\nWHERE doc @? '$.a[*] ? (@ > 2)'" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY[1, 2, 3]", + "formatted": "SELECT ARRAY[1, 2, 3]" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY(SELECT 1)", + "formatted": "SELECT\n ARRAY(SELECT\n 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg') IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg') IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\xDEADBEEF'", + "formatted": "SELECT e'\\xDEADBEEF'" + }, + { + "dialect": "postgres", + "sql": "SELECT CAST(e'\\176' AS BYTEA)", + "formatted": "SELECT CAST(e'\\176' AS BYTEA)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')", + "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '...$') IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(' X' FROM ' XXX ')", + "formatted": "SELECT TRIM(' X' FROM ' XXX ')" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)", + "formatted": "SELECT TRIM(LEADING 'bla' FROM ' XXX ' COLLATE utf8_bin)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)", + "formatted": "SELECT *\nFROM JSON_TO_RECORDSET(z) AS y(\"rank\" INT)" + }, + { + "dialect": "postgres", + "sql": "SELECT ~x", + "formatted": "SELECT ~x" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM r CROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)", + "formatted": "SELECT *\nFROM r\nCROSS JOIN LATERAL UNNEST(ARRAY[1]) AS s(location)" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_SCHEMA", + "formatted": "SELECT CURRENT_SCHEMA" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_USER", + "formatted": "SELECT CURRENT_USER" + }, + { + "dialect": "postgres", + "sql": "SELECT CURRENT_ROLE", + "formatted": "SELECT CURRENT_ROLE" + }, + { + "dialect": "postgres", + "sql": "SELECT VERSION()", + "formatted": "SELECT VERSION()" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ONLY t1", + "formatted": "SELECT *\nFROM ONLY t1" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '-1 MONTH'", + "formatted": "SELECT INTERVAL '-1 MONTH'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '4.1 DAY'", + "formatted": "SELECT INTERVAL '4.1 DAY'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '3.14159 HOUR'", + "formatted": "SELECT INTERVAL '3.14159 HOUR'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '2.5 MONTH'", + "formatted": "SELECT INTERVAL '2.5 MONTH'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '-10.75 MINUTE'", + "formatted": "SELECT INTERVAL '-10.75 MINUTE'" + }, + { + "dialect": "postgres", + "sql": "SELECT INTERVAL '0.123456789 SECOND'", + "formatted": "SELECT INTERVAL '0.123456789 SECOND'" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '30' FROM t", + "formatted": "SELECT date_col - INTERVAL '30'\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '1' AS one_second_later", + "formatted": "SELECT date_col - INTERVAL '1' AS one_second_later" + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '30' DAY FROM t", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT date_col - INTERVAL '1' HOUR AS one_hour_later", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total", + "formatted": "SELECT SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - SUM(x) OVER (PARTITION BY y ORDER BY interval ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS total" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)", + "formatted": "SELECT *\nFROM test_data, LATERAL JSONB_ARRAY_ELEMENTS(data) WITH ORDINALITY AS elem(value, ordinality)" + }, + { + "dialect": "postgres", + "sql": "SELECT id, name FROM xml_data AS t, XMLTABLE('/root/user' PASSING t.xml COLUMNS id INT PATH '@id', name TEXT PATH 'name/text()') AS x", + "formatted": "SELECT\n id,\n name\nFROM xml_data AS t, XMLTABLE(\n '/root/user'\n PASSING\n t.xml\n COLUMNS\n id INT PATH '@id',\n name TEXT PATH 'name/text()'\n) AS x" + }, + { + "dialect": "postgres", + "sql": "SELECT id, value FROM xml_content AS t, XMLTABLE(XMLNAMESPACES('http://example.com/ns1' AS ns1, 'http://example.com/ns2' AS ns2), '/root/data' PASSING t.xml COLUMNS id INT PATH '@ns1:id', value TEXT PATH 'ns2:value/text()') AS x", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t WHERE some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE", + "formatted": "SELECT *\nFROM t\nWHERE\n some_column >= CURRENT_DATE + INTERVAL '1 day 1 hour' AND some_another_column IS TRUE" + }, + { + "dialect": "postgres", + "sql": "WITH t1 AS MATERIALIZED (SELECT 1), t2 AS NOT MATERIALIZED (SELECT 2) SELECT * FROM t1, t2", + "formatted": "WITH t1 AS MATERIALIZED (\n SELECT 1\n),\n\nt2 AS NOT MATERIALIZED (\n SELECT 2\n)\n\nSELECT *\nFROM t1, t2" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)", + "formatted": "SELECT *\nFROM JSON_ARRAY_ELEMENTS('[1,true, [2,false]]') WITH ORDINALITY AS kv_json(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER a, SUM(y) OVER b FROM c WINDOW a AS (PARTITION BY d), b AS (PARTITION BY e)", + "formatted": "SELECT\n SUM(x) OVER a,\n SUM(y) OVER b\nFROM c\nWINDOW\n a AS (PARTITION BY d),\n b AS (PARTITION BY e)" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1) IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT CASE WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab') THEN 1 ELSE 0 END", + "formatted": "SELECT\n CASE\n WHEN SUBSTRING('abcdefg' FROM 1 FOR 2) IN ('ab')\n THEN 1\n ELSE 0\n END" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM \"x\" WHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')", + "formatted": "SELECT *\nFROM \"x\"\nWHERE SUBSTRING(\"x\".\"foo\" FROM 1 FOR 2) IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM x WHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')", + "formatted": "SELECT *\nFROM x\nWHERE SUBSTRING('Thomas' FROM '%#\"o_a#\"_' FOR '#') IN ('mas')" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))", + "formatted": "SELECT SUBSTRING('bla' + 'foo' || 'bar' FROM 3 - 1 + 5 FOR 4 + SOME_FUNC(arg1, arg2))" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP(1284352323.5), TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')", + "formatted": "SELECT\n TO_TIMESTAMP(1284352323.5),\n TO_TIMESTAMP('05 Dec 2000', 'DD Mon YYYY')" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')", + "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 AM', 'DD Mon YYYY HH:MI AM')" + }, + { + "dialect": "postgres", + "sql": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')", + "formatted": "SELECT TO_TIMESTAMP('05 Dec 2000 10:00 PM', 'DD Mon YYYY HH:MI PM')" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo, LATERAL (SELECT * FROM bar WHERE bar.id = foo.bar_id) AS ss", + "formatted": "SELECT *\nFROM foo, LATERAL (\n SELECT *\n FROM bar\n WHERE bar.id = foo.bar_id\n) AS ss" + }, + { + "dialect": "postgres", + "sql": "SELECT c.oid, n.nspname, c.relname FROM pg_catalog.pg_class AS c LEFT JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid) ORDER BY 2, 3", + "formatted": "SELECT\n c.oid,\n n.nspname,\n c.relname\nFROM pg_catalog.pg_class AS c\nLEFT JOIN pg_catalog.pg_namespace AS n\n ON n.oid = c.relnamespace\nWHERE\n c.relname OPERATOR(pg_catalog.~) '^(courses)$' COLLATE pg_catalog.default AND pg_catalog.PG_TABLE_IS_VISIBLE(c.oid)\nORDER BY\n 2,\n 3" + }, + { + "dialect": "postgres", + "sql": "SELECT date_add(current_date, interval '7' day)", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT e'foo \\' bar'", + "formatted": "SELECT e'foo \\' bar'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\n'", + "formatted": "SELECT e'\\n'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'\\t'", + "formatted": "SELECT e'\\t'" + }, + { + "dialect": "postgres", + "sql": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x FROM tab", + "formatted": "SELECT e'update table_name set a = \\'foo\\' where 1 = 0' AS x\nFROM tab" + }, + { + "dialect": "postgres", + "sql": "select count() OVER(partition by a order by a range offset preceding exclude current row)", + "formatted": "SELECT COUNT() OVER (PARTITION BY a ORDER BY a range offset preceding exclude CURRENT ROW)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)", + "formatted": "SELECT SUBSTRING('Thomas' FOR 3 FROM 2)" + }, + { + "dialect": "postgres", + "sql": "SELECT DATE_PART('isodow'::varchar(6), current_date)", + "formatted": "SELECT DATE_PART('isodow'::varchar(6), CURRENT_DATE)" + }, + { + "dialect": "postgres", + "sql": "SELECT $$a$$", + "formatted": "SELECT $$a$$" + }, + { + "dialect": "postgres", + "sql": "SELECT $$Dianne's order$$", + "formatted": "SELECT $$Dianne's order$$" + }, + { + "dialect": "postgres", + "sql": "SELECT $$The price is $9.95$$ AS msg", + "formatted": "SELECT $$The price is $9.95$$ AS msg" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT LIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT LIKE ANY(ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT LIKE ANY(\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT ILIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT ILIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT 'testa 1' NOT ILIKE ANY(ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n 'testa 1' NOT ILIKE ANY(\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT 'testa 1' LIKE ALL (ARRAY['testa%', 'testb%'])", + "formatted": "SELECT\n NOT 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT ('testa 1' LIKE ALL (ARRAY['testa%', 'testb%']))", + "formatted": "SELECT\n NOT (\n 'testa 1' LIKE ALL (\n ARRAY['testa%', 'testb%']\n )\n )" + }, + { + "dialect": "postgres", + "sql": "SELECT to_timestamp(123)::time without time zone", + "formatted": "SELECT to_timestamp(123)::time without time zone" + }, + { + "dialect": "postgres", + "sql": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)", + "formatted": "SELECT SUM(x) OVER (PARTITION BY a ORDER BY d ROWS 1 PRECEDING)" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)", + "formatted": "SELECT SUBSTRING(2022::CHAR(4) || LPAD(3::CHAR(2), 2, '0') FROM 3 FOR 4)" + }, + { + "dialect": "postgres", + "sql": "SELECT m.name FROM manufacturers AS m LEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname ON TRUE WHERE pname IS NULL", + "formatted": "SELECT m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) pname\n ON TRUE\nWHERE pname IS NULL", + "previously_formatted": "SELECT\n m.name\nFROM manufacturers AS m\nLEFT JOIN LATERAL GET_PRODUCT_NAMES(m.id) AS pname\n ON TRUE\nWHERE\n pname IS NULL" + }, + { + "dialect": "postgres", + "sql": "SELECT p1.id, p2.id, v1, v2 FROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2 WHERE (v1 <-> v2) < 10 AND p1.id <> p2.id", + "formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) v1, LATERAL VERTICES(p2.poly) v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id", + "previously_formatted": "SELECT\n p1.id,\n p2.id,\n v1,\n v2\nFROM polygons AS p1, polygons AS p2, LATERAL VERTICES(p1.poly) AS v1, LATERAL VERTICES(p2.poly) AS v2\nWHERE\n (\n v1 <-> v2\n ) < 10 AND p1.id <> p2.id" + }, + { + "dialect": "postgres", + "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE deleted NOTNULL", + "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE deleted NOTNULL" + }, + { + "dialect": "postgres", + "sql": "SELECT id, email, CAST(deleted AS TEXT) FROM users WHERE NOT deleted ISNULL", + "formatted": "SELECT\n id,\n email,\n CAST(deleted AS TEXT)\nFROM users\nWHERE NOT deleted ISNULL" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_ARRAY_ELEMENTS((foo->'sections')::JSON) AS sections", + "formatted": "SELECT\n JSON_ARRAY_ELEMENTS((\n foo -> 'sections'\n )::JSON) AS sections" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM t1*", + "formatted": "SELECT *\nFROM t1*" + }, + { + "dialect": "postgres", + "sql": "SELECT SUBSTRING('afafa' for 1)", + "formatted": "SELECT SUBSTRING('afafa' FOR 1)" + }, + { + "dialect": "postgres", + "sql": "\n WITH\n json_data AS (SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data),\n field_ids AS (SELECT 'field_id' AS field_id)\n\n SELECT\n JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\n FROM json_data, field_ids\n ", + "formatted": "WITH json_data AS (\n SELECT '{\"field_id\": [1, 2, 3]}'::JSON AS data\n),\n\nfield_ids AS (\n SELECT 'field_id' AS field_id\n)\n\nSELECT JSON_ARRAY_ELEMENTS(json_data.data -> field_ids.field_id) AS element\nFROM json_data, field_ids" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")", + "formatted": "SELECT TRIM(LEADING ' XXX ' COLLATE \"de_DE\")" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")", + "formatted": "SELECT TRIM(TRAILING ' XXX ' COLLATE \"de_DE\")" + }, + { + "dialect": "postgres", + "sql": "SELECT js, js IS JSON AS \"json?\", js IS JSON VALUE AS \"scalar?\", js IS JSON SCALAR AS \"scalar?\", js IS JSON OBJECT AS \"object?\", js IS JSON ARRAY AS \"array?\" FROM t", + "formatted": "SELECT\n js,\n js IS JSON AS \"json?\",\n js IS JSON VALUE AS \"scalar?\",\n js IS JSON SCALAR AS \"scalar?\",\n js IS JSON OBJECT AS \"object?\",\n js IS JSON ARRAY AS \"array?\"\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT js, js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\", js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\", js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\" FROM t", + "formatted": "SELECT\n js,\n js IS JSON ARRAY WITH UNIQUE KEYS AS \"array w. UK?\",\n js IS JSON ARRAY WITHOUT UNIQUE KEYS AS \"array w/o UK?\",\n js IS JSON ARRAY UNIQUE KEYS AS \"array w UK 2?\"\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT 1 FROM ((VALUES (1)) AS vals(id) LEFT OUTER JOIN tbl ON vals.id = tbl.id)", + "formatted": "SELECT 1\nFROM ((VALUES (1)) AS vals(id)\nLEFT OUTER JOIN tbl\n ON vals.id = tbl.id)" + }, + { + "dialect": "postgres", + "sql": "SELECT OVERLAY(a PLACING b FROM 1)", + "formatted": "SELECT OVERLAY(a PLACING b FROM 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)", + "formatted": "SELECT OVERLAY(a PLACING b FROM 1 FOR 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value FROM (VALUES (0, 't')) AS fake_data(a, b)", + "formatted": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY a) FILTER(WHERE CAST(b AS BOOLEAN)) AS mean_value\nFROM (VALUES (0, 't')) AS fake_data(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %s", + "formatted": "SELECT *\nFROM foo\nWHERE id = %s" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = %(id_param)s", + "formatted": "SELECT *\nFROM foo\nWHERE id = %(id_param)s" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM foo WHERE id = ?", + "formatted": "SELECT *\nFROM foo\nWHERE id = ?" + }, + { + "dialect": "postgres", + "sql": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| array['b', 'c']", + "formatted": "SELECT '{\"a\":1, \"b\":2, \"c\":3}'::jsonb ?| ARRAY['b', 'c']" + }, + { + "dialect": "postgres", + "sql": "SELECT '[\"a\", \"b\"]'::jsonb ?& array['a', 'b']", + "formatted": "SELECT '[\"a\", \"b\"]'::jsonb ?& ARRAY['a', 'b']" + }, + { + "dialect": "postgres", + "sql": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'", + "formatted": "SELECT '[\"a\", {\"b\":1}]'::jsonb #- '{1,b}'" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(DISTINCT name) FROM users", + "formatted": "SELECT JSON_AGG(DISTINCT name)\nFROM users" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", + "formatted": "SELECT JSON_AGG(c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" + }, + { + "dialect": "postgres", + "sql": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1) FROM (VALUES ('c'), ('b'), ('a')) AS t(c1)", + "formatted": "SELECT JSON_AGG(DISTINCT c1 ORDER BY c1)\nFROM (VALUES ('c'), ('b'), ('a')) AS t(c1)" + }, + { + "dialect": "postgres", + "sql": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)", + "formatted": "SELECT NUMRANGE(1.1, 2.2) -|- NUMRANGE(2.2, 3.3)" + }, + { + "dialect": "postgres", + "sql": "SELECT SLOPE(point '(4,4)', point '(0,0)')", + "formatted": "SELECT SLOPE(point '(4,4)', point '(0,0)')" + }, + { + "dialect": "postgres", + "sql": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])", + "formatted": "SELECT MLEAST(VARIADIC ARRAY[10, -1, 5, 4.4])" + }, + { + "dialect": "postgres", + "sql": "SELECT MLEAST(VARIADIC ARRAY[]::numeric[])", + "formatted": null + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM schema_name.table_name st WHERE JSON_EXTRACT_PATH_TEXT((st.data)::json, variadic array['test'::text]) = 'test'::text", + "formatted": "SELECT *\nFROM schema_name.table_name st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text", + "previously_formatted": "SELECT\n *\nFROM schema_name.table_name AS st\nWHERE\n JSON_EXTRACT_PATH_TEXT(\n (\n st.data\n )::json,\n VARIADIC ARRAY['test'::text]\n ) = 'test'::text" + }, + { + "dialect": "postgres", + "sql": "WITH t(c) AS (SELECT 1) SELECT * INTO UNLOGGED foo FROM (SELECT c AS c FROM t) AS temp", + "formatted": "WITH t(c) AS (\n SELECT 1\n)\n\nSELECT *\nINTO UNLOGGED foo\nFROM (\n SELECT c AS c\n FROM t\n) AS temp" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)", + "formatted": "SELECT *\nFROM UNNEST(ARRAY[1, 2], ARRAY['foo', 'bar', 'baz']) AS x(a, b)" + }, + { + "dialect": "postgres", + "sql": "SELECT TRIM(ARRAY_TO_STRING(ARRAY(SELECT val FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)), ' '))", + "formatted": "SELECT\n TRIM(ARRAY_TO_STRING(\n ARRAY(\n SELECT val\n FROM UNNEST(ARRAY['a', 'b']) WITH ORDINALITY AS u(val, rn)\n ),\n ' '\n ))" + }, + { + "dialect": "postgres", + "sql": "SELECT a, LOGICAL_OR(b) FROM table GROUP BY a", + "formatted": "SELECT\n a,\n LOGICAL_OR(b)\nFROM table\nGROUP BY\n a" + }, + { + "dialect": "postgres", + "sql": "SELECT CONCAT('abcde', 2, NULL, 22)", + "formatted": "SELECT CONCAT('abcde', 2, NULL, 22)" + }, + { + "dialect": "postgres", + "sql": "SELECT u & 5 FROM t", + "formatted": "SELECT u & 5\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT (U&'\\FE01' || 'Test literal') AS label FROM data", + "formatted": "SELECT\n (\n U&'\\FE01' || 'Test literal'\n ) AS label\nFROM data" + }, + { + "dialect": "postgres", + "sql": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label", + "formatted": "SELECT U&'d!0061t!+000061' UESCAPE '!' AS label" + }, + { + "dialect": "postgres", + "sql": "SELECT u&'\\0441\\043B\\043E\\043D'", + "formatted": "SELECT u&'\\0441\\043B\\043E\\043D'" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ROWS FROM (FUNC1(col1, col2))", + "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1, col2))" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY", + "formatted": "SELECT *\nFROM ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT), FUNC2(col2) AS alias2(\"col2\" INT)) WITH ORDINALITY" + }, + { + "dialect": "postgres", + "sql": "SELECT * FROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)", + "formatted": "SELECT *\nFROM table1, ROWS FROM (FUNC1(col1) AS alias1(\"col1\" TEXT)) WITH ORDINALITY AS alias3(\"col3\" INT, \"col4\" TEXT)" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)", + "formatted": "SELECT ARRAY_LENGTH(ARRAY[1, 2, 3], 1)" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo)", + "formatted": "SELECT XMLELEMENT(NAME foo)" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar))" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b)) FROM test", + "formatted": "SELECT XMLELEMENT(NAME test, XMLATTRIBUTES(a, b))\nFROM test" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES(CURRENT_DATE AS bar), 'cont', 'ent')" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))", + "formatted": "SELECT XMLELEMENT(NAME \"foo$bar\", XMLATTRIBUTES('xyz' AS \"a&b\"))" + }, + { + "dialect": "postgres", + "sql": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))", + "formatted": "SELECT XMLELEMENT(NAME foo, XMLATTRIBUTES('xyz' AS bar), XMLELEMENT(NAME abc), XMLCOMMENT('test'), XMLELEMENT(NAME xyz))" + }, + { + "dialect": "postgres", + "sql": "WITH RECURSIVE search_graph(id, link, data, depth) AS (SELECT g.id, g.link, g.data, 1 FROM graph AS g UNION ALL SELECT g.id, g.link, g.data, sg.depth + 1 FROM graph AS g, search_graph AS sg WHERE g.id = sg.link) CYCLE id SET is_cycle USING path SELECT * FROM search_graph", + "formatted": "WITH RECURSIVE search_graph(id, link, data, depth) AS (\n SELECT\n g.id,\n g.link,\n g.data,\n 1\n FROM graph AS g\n UNION ALL\n SELECT\n g.id,\n g.link,\n g.data,\n sg.depth + 1\n FROM graph AS g, search_graph AS sg\n WHERE g.id = sg.link\n)\n\nCYCLE id SET is_cycle USING path\nSELECT *\nFROM search_graph" + }, + { + "dialect": "postgres", + "sql": "SELECT a <<->> b", + "formatted": "SELECT a <<->> b" + }, + { + "dialect": "postgres", + "sql": "SELECT r IS NOT NULL FROM t", + "formatted": "SELECT r IS NOT NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT r IS NULL FROM t", + "formatted": "SELECT NOT r IS NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT NOT r IS NOT NULL FROM t", + "formatted": "SELECT NOT r IS NOT NULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT r NOTNULL FROM t", + "formatted": "SELECT r NOTNULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT r ISNULL FROM t", + "formatted": "SELECT r ISNULL\nFROM t" + }, + { + "dialect": "postgres", + "sql": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]", + "formatted": "SELECT ARRAY[1, 2, 3] <@ ARRAY[1, 2]" + }, + { + "dialect": "snowflake", + "sql": "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", + "formatted": "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data:level1:level2:level3::VARIANT\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM x ASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", + "formatted": "SELECT *\nFROM x\nASOF JOIN y OFFSET MATCH_CONDITION (x.a > y.a)", + "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS OFFSET MATCH_CONDITION (x.a > y.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM x ASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", + "formatted": "SELECT *\nFROM x\nASOF JOIN y LIMIT MATCH_CONDITION (x.a > y.a)", + "previously_formatted": "SELECT\n *\nFROM x\nASOF JOIN y AS LIMIT MATCH_CONDITION (x.a > y.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT session", + "formatted": "SELECT session" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a", + "formatted": "SELECT DATE_PART(EPOCH_MILLISECOND, CURRENT_TIMESTAMP()) AS a" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET(a, b)", + "formatted": "SELECT GET(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HASH_AGG(a, b, c, d)", + "formatted": "SELECT HASH_AGG(a, b, c, d)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GREATEST(1, 2, 3, NULL)", + "formatted": "SELECT GREATEST(1, 2, 3, NULL)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)", + "formatted": "SELECT GREATEST_IGNORE_NULLS(1, 2, 3, NULL)" + }, + { + "dialect": "snowflake", + "sql": "SELECT LEAST(5, NULL, 7, 3)", + "formatted": "SELECT LEAST(5, NULL, 7, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)", + "formatted": "SELECT LEAST_IGNORE_NULLS(5, NULL, 7, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAX(x)", + "formatted": "SELECT MAX(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COUNT(x)", + "formatted": "SELECT COUNT(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MIN(amount)", + "formatted": "SELECT MIN(amount)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MODE(x)", + "formatted": "SELECT MODE(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MODE(status) OVER (PARTITION BY region) FROM orders", + "formatted": "SELECT MODE(status) OVER (PARTITION BY region)\nFROM orders" + }, + { + "dialect": "snowflake", + "sql": "SELECT TAN(x)", + "formatted": "SELECT TAN(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COS(x)", + "formatted": "SELECT COS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SINH(1.5)", + "formatted": "SELECT SINH(1.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MOD(x, y)", + "formatted": "SELECT MOD(x, y)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(x)", + "formatted": "SELECT ROUND(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(123.456, -1)", + "formatted": "SELECT ROUND(123.456, -1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')", + "formatted": "SELECT ROUND(123.456, 2, 'HALF_AWAY_FROM_ZERO')" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(x)", + "formatted": "SELECT FLOOR(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(135.135, 1)", + "formatted": "SELECT FLOOR(135.135, 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FLOOR(x, -1)", + "formatted": "SELECT FLOOR(x, -1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary) FROM employees", + "formatted": "SELECT PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY salary)\nFROM employees" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K(col) FROM t", + "formatted": "SELECT APPROX_TOP_K(col)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K(category, 3) FROM t", + "formatted": "SELECT APPROX_TOP_K(category, 3)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, col)", + "formatted": "SELECT MINHASH(5, col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, col1, col2)", + "formatted": "SELECT MINHASH(5, col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH(5, *)", + "formatted": "SELECT MINHASH(5, *)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINHASH_COMBINE(minhash_col)", + "formatted": "SELECT MINHASH_COMBINE(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROXIMATE_SIMILARITY(minhash_col)", + "formatted": "SELECT APPROXIMATE_SIMILARITY(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)", + "formatted": "SELECT APPROXIMATE_JACCARD_INDEX(minhash_col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)", + "formatted": "SELECT APPROX_PERCENTILE_ACCUMULATE(col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)", + "formatted": "SELECT APPROX_PERCENTILE_ESTIMATE(state, 0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)", + "formatted": "SELECT APPROX_TOP_K_ACCUMULATE(col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_COMBINE(state, 2)", + "formatted": "SELECT APPROX_TOP_K_COMBINE(state, 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_COMBINE(state)", + "formatted": "SELECT APPROX_TOP_K_COMBINE(state)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)", + "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column, 4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_TOP_K_ESTIMATE(state_column)", + "formatted": "SELECT APPROX_TOP_K_ESTIMATE(state_column)" + }, + { + "dialect": "snowflake", + "sql": "SELECT APPROX_PERCENTILE_COMBINE(state_column)", + "formatted": "SELECT APPROX_PERCENTILE_COMBINE(state_column)" + }, + { + "dialect": "snowflake", + "sql": "SELECT EQUAL_NULL(1, 2)", + "formatted": "SELECT EQUAL_NULL(1, 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT FACTORIAL(5)", + "formatted": "SELECT FACTORIAL(5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_LENGTH('abc')", + "formatted": "SELECT BIT_LENGTH('abc')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_LENGTH(x'A1B2')", + "formatted": "SELECT BIT_LENGTH(x'A1B2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_BUCKET_NUMBER(32769)", + "formatted": "SELECT BITMAP_BUCKET_NUMBER(32769)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_CONSTRUCT_AGG(value)", + "formatted": "SELECT BITMAP_CONSTRUCT_AGG(value)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value)) FROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))", + "formatted": "SELECT BITMAP_COUNT(BITMAP_CONSTRUCT_AGG(value))\nFROM TABLE(FLATTEN(INPUT => ARRAY_CONSTRUCT(1, 2, 3, 5)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLAND(1, -2)", + "formatted": "SELECT BOOLAND(1, -2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLXOR(2, 0)", + "formatted": "SELECT BOOLXOR(2, 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLOR(1, 0)", + "formatted": "SELECT BOOLOR(1, 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_BOOLEAN('true')", + "formatted": "SELECT TO_BOOLEAN('true')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_BOOLEAN(1)", + "formatted": "SELECT TO_BOOLEAN(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_VARIANT(123)", + "formatted": "SELECT TO_VARIANT(123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))", + "formatted": "SELECT IS_NULL_VALUE(GET_PATH(payload, 'field'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT RTRIMMED_LENGTH(' ABCD ')", + "formatted": "SELECT RTRIMMED_LENGTH(' ABCD ')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_DECODE_STRING('48656C6C6F')", + "formatted": "SELECT HEX_DECODE_STRING('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World')", + "formatted": "SELECT HEX_ENCODE('Hello World')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World', 1)", + "formatted": "SELECT HEX_ENCODE('Hello World', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HEX_ENCODE('Hello World', 0)", + "formatted": "SELECT HEX_ENCODE('Hello World', 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT IFNULL(col1, col2)", + "formatted": "SELECT IFNULL(col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')", + "formatted": "SELECT NEXT_DAY('2025-10-15', 'FRIDAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT NVL2(col1, col2, col3)", + "formatted": "SELECT NVL2(col1, col2, col3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NVL(col1, col2)", + "formatted": "SELECT NVL(col1, col2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHR(8364)", + "formatted": "SELECT CHR(8364)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_JSON('{\"key\": \"value\"}')", + "formatted": "SELECT CHECK_JSON('{\"key\": \"value\"}')" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_XML('value')", + "formatted": "SELECT CHECK_XML('value')" + }, + { + "dialect": "snowflake", + "sql": "SELECT CHECK_XML('value', TRUE)", + "formatted": "SELECT CHECK_XML('value', TRUE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COMPRESS('Hello World', 'ZLIB')", + "formatted": "SELECT COMPRESS('Hello World', 'ZLIB')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')", + "formatted": "SELECT DECOMPRESS_BINARY('compressed_data', 'SNAPPY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')", + "formatted": "SELECT DECOMPRESS_STRING('compressed_data', 'ZSTD')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LPAD('Hello', 10, '*')", + "formatted": "SELECT LPAD('Hello', 10, '*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LPAD(tbl.bin_col, 10)", + "formatted": "SELECT LPAD(tbl.bin_col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD('Hello', 10, '*')", + "formatted": "SELECT RPAD('Hello', 10, '*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD(tbl.bin_col, 10)", + "formatted": "SELECT RPAD(tbl.bin_col, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + "formatted": "SELECT\n RPAD(\n TO_BINARY('Hi', 'UTF8'),\n 10,\n TO_BINARY('_', 'UTF8')\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT SOUNDEX(column_name)", + "formatted": "SELECT SOUNDEX(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SOUNDEX_P123(column_name)", + "formatted": "SELECT SOUNDEX_P123(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ABS(x)", + "formatted": "SELECT ABS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ASIN(0.5)", + "formatted": "SELECT ASIN(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ASINH(0.5)", + "formatted": "SELECT ASINH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATAN(0.5)", + "formatted": "SELECT ATAN(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATAN2(0.5, 0.3)", + "formatted": "SELECT ATAN2(0.5, 0.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ATANH(0.5)", + "formatted": "SELECT ATANH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CBRT(27.0)", + "formatted": "SELECT CBRT(27.0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT POW(2, 3)", + "formatted": "SELECT POW(2, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT POW(2.5, 3.0)", + "formatted": "SELECT POW(2.5, 3.0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SQUARE(2.5)", + "formatted": "SELECT SQUARE(2.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SIGN(x)", + "formatted": "SELECT SIGN(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT COSH(1.5)", + "formatted": "SELECT COSH(1.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TANH(0.5)", + "formatted": "SELECT TANH(0.5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRANSLATE(column_name, 'abc', '123')", + "formatted": "SELECT TRANSLATE(column_name, 'abc', '123')" + }, + { + "dialect": "snowflake", + "sql": "SELECT UNICODE(column_name)", + "formatted": "SELECT UNICODE(column_name)" + }, + { + "dialect": "snowflake", + "sql": "SELECT WIDTH_BUCKET(col, 0, 100, 10)", + "formatted": "SELECT WIDTH_BUCKET(col, 0, 100, 10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT PI()", + "formatted": "SELECT PI()" + }, + { + "dialect": "snowflake", + "sql": "SELECT DEGREES(PI() / 3)", + "formatted": "SELECT DEGREES(PI() / 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DEGREES(1)", + "formatted": "SELECT DEGREES(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RADIANS(180)", + "formatted": "SELECT RADIANS(180)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_AVGX(y, x)", + "formatted": "SELECT REGR_AVGX(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_AVGY(y, x)", + "formatted": "SELECT REGR_AVGY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_COUNT(y, x)", + "formatted": "SELECT REGR_COUNT(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_INTERCEPT(y, x)", + "formatted": "SELECT REGR_INTERCEPT(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_R2(y, x)", + "formatted": "SELECT REGR_R2(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SXX(y, x)", + "formatted": "SELECT REGR_SXX(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SXY(y, x)", + "formatted": "SELECT REGR_SXY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SYY(y, x)", + "formatted": "SELECT REGR_SYY(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGR_SLOPE(y, x)", + "formatted": "SELECT REGR_SLOPE(y, x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RANDSTR(123, 456)", + "formatted": "SELECT RANDSTR(123, 456)" + }, + { + "dialect": "snowflake", + "sql": "SELECT RANDSTR(123, RANDOM())", + "formatted": "SELECT RANDSTR(123, RANDOM())" + }, + { + "dialect": "snowflake", + "sql": "SELECT NORMAL(0, 1, RANDOM())", + "formatted": "SELECT NORMAL(0, 1, RANDOM())" + }, + { + "dialect": "snowflake", + "sql": "SELECT GROUPING_ID(a, b) AS g_id FROM x GROUP BY ROLLUP (a, b)", + "formatted": "SELECT GROUPING_ID(a, b) AS g_id\nFROM x\nGROUP BY\n ROLLUP (\n a,\n b\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT XMLGET(object_col, 'level2')", + "formatted": "SELECT XMLGET(object_col, 'level2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT XMLGET(object_col, 'level3', 1)", + "formatted": "SELECT XMLGET(object_col, 'level3', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT {*} FROM my_table", + "formatted": "SELECT {*}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {my_table.*} FROM my_table", + "formatted": "SELECT {my_table.*}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* ILIKE 'col1%'} FROM my_table", + "formatted": "SELECT {* ILIKE 'col1%'}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* EXCLUDE (col1)} FROM my_table", + "formatted": "SELECT {* EXCLUDE (col1)}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT {* EXCLUDE (col1, col2)} FROM my_table", + "formatted": "SELECT {* EXCLUDE (col1, col2)}\nFROM my_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT a, b, COUNT(*) FROM x GROUP BY ALL LIMIT 100", + "formatted": "SELECT\n a,\n b,\n COUNT(*)\nFROM x\nGROUP BY ALL\nLIMIT 100" + }, + { + "dialect": "snowflake", + "sql": "SELECT STAR(tbl, exclude := [foo])", + "formatted": "SELECT STAR(tbl, exclude := [foo])" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))", + "formatted": "SELECT CAST([1, 2, 3] AS VECTOR(FLOAT, 3))" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_COSINE_SIMILARITY(a, b)", + "formatted": "SELECT VECTOR_COSINE_SIMILARITY(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_INNER_PRODUCT(a, b)", + "formatted": "SELECT VECTOR_INNER_PRODUCT(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_L1_DISTANCE(a, b)", + "formatted": "SELECT VECTOR_L1_DISTANCE(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT VECTOR_L2_DISTANCE(a, b)", + "formatted": "SELECT VECTOR_L2_DISTANCE(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CONNECT_BY_ROOT test AS test_column_alias", + "formatted": "SELECT CONNECT_BY_ROOT test AS test_column_alias" + }, + { + "dialect": "snowflake", + "sql": "SELECT rename, replace", + "formatted": "SELECT\n rename,\n replace" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))", + "formatted": "SELECT TIMEADD(HOUR, 2, CAST('09:05:03' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))", + "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS MAP(VARCHAR, INT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_CAT(CAST(col AS MAP(VARCHAR, VARCHAR)), CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_CONTAINS_KEY('k1', CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')", + "formatted": "SELECT MAP_DELETE(CAST(col AS MAP(VARCHAR, VARCHAR)), 'k1')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')", + "formatted": "SELECT MAP_INSERT(CAST(col AS MAP(VARCHAR, VARCHAR)), 'b', '2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_KEYS(CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')", + "formatted": "SELECT MAP_PICK(CAST(col AS MAP(VARCHAR, VARCHAR)), 'a', 'c')" + }, + { + "dialect": "snowflake", + "sql": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))", + "formatted": "SELECT MAP_SIZE(CAST(col AS MAP(VARCHAR, VARCHAR)))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))", + "formatted": "SELECT CAST(OBJECT_CONSTRUCT('a', 1) AS OBJECT(a CHAR NOT NULL))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST([1, 2, 3] AS ARRAY(INT))", + "formatted": "SELECT CAST([1, 2, 3] AS ARRAY(INT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)", + "formatted": "SELECT CAST(obj AS OBJECT(x CHAR) RENAME FIELDS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)", + "formatted": "SELECT CAST(obj AS OBJECT(x CHAR, y VARCHAR) ADD FIELDS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_NTZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_NTZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_LTZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_LTZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP_TZ(x) FROM t", + "formatted": "SELECT TO_TIMESTAMP_TZ(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)", + "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)", + "formatted": "SELECT TIMESTAMP_FROM_PARTS(2024, 5, 9, 14, 30, 45, 123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMP_LTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')", + "formatted": "SELECT TIMESTAMP_TZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 0, 'America/Los_Angeles')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + "formatted": "SELECT\n TIMESTAMP_FROM_PARTS(\n CAST('2024-05-09' AS DATE),\n CAST('14:30:45' AS TIME)\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))", + "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(TO_DATE('2013-04-05'), TO_TIME('12:00:00'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)", + "formatted": "SELECT TIMESTAMP_NTZ_FROM_PARTS(2013, 4, 5, 12, 00, 00, 987654321)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_FROM_PARTS(1977, 8, 7)", + "formatted": "SELECT DATE_FROM_PARTS(1977, 8, 7)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(v, 'attr[0].name') FROM vartab", + "formatted": "SELECT GET_PATH(v, 'attr[0].name')\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_ARRAY(CAST(x AS ARRAY))", + "formatted": "SELECT TO_ARRAY(CAST(x AS ARRAY))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))", + "formatted": "SELECT TO_ARRAY(CAST(['test'] AS VARIANT))" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_UNIQUE_AGG(x)", + "formatted": "SELECT ARRAY_UNIQUE_AGG(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_APPEND([1, 2, 3], 4)", + "formatted": "SELECT ARRAY_APPEND([1, 2, 3], 4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CAT([1, 2], [3, 4])", + "formatted": "SELECT ARRAY_CAT([1, 2], [3, 4])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_PREPEND([2, 3, 4], 1)", + "formatted": "SELECT ARRAY_PREPEND([2, 3, 4], 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_REMOVE([1, 2, 3], 2)", + "formatted": "SELECT ARRAY_REMOVE([1, 2, 3], 2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAYS_ZIP([1, 2, 3])", + "formatted": "SELECT ARRAYS_ZIP([1, 2, 3])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + "formatted": "SELECT\n ARRAYS_ZIP(\n [1, 2, 3],\n ['a', 'b', 'c'],\n [10, 20, 30]\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_AGG(review, 'Summarize the reviews')", + "formatted": "SELECT AI_AGG(review, 'Summarize the reviews')" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_SUMMARIZE_AGG(review)", + "formatted": "SELECT AI_SUMMARIZE_AGG(review)" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])", + "formatted": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'])" + }, + { + "dialect": "snowflake", + "sql": "SELECT OBJECT_CONSTRUCT()", + "formatted": "SELECT OBJECT_CONSTRUCT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ACCOUNT()", + "formatted": "SELECT CURRENT_ACCOUNT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ACCOUNT_NAME()", + "formatted": "SELECT CURRENT_ACCOUNT_NAME()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_AVAILABLE_ROLES()", + "formatted": "SELECT CURRENT_AVAILABLE_ROLES()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_CLIENT()", + "formatted": "SELECT CURRENT_CLIENT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_IP_ADDRESS()", + "formatted": "SELECT CURRENT_IP_ADDRESS()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_DATABASE()", + "formatted": "SELECT CURRENT_DATABASE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SCHEMAS()", + "formatted": "SELECT CURRENT_SCHEMAS()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SECONDARY_ROLES()", + "formatted": "SELECT CURRENT_SECONDARY_ROLES()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SESSION()", + "formatted": "SELECT CURRENT_SESSION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_STATEMENT()", + "formatted": "SELECT CURRENT_STATEMENT()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_VERSION()", + "formatted": "SELECT CURRENT_VERSION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_TRANSACTION()", + "formatted": "SELECT CURRENT_TRANSACTION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_WAREHOUSE()", + "formatted": "SELECT CURRENT_WAREHOUSE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ORGANIZATION_USER()", + "formatted": "SELECT CURRENT_ORGANIZATION_USER()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_REGION()", + "formatted": "SELECT CURRENT_REGION()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ROLE()", + "formatted": "SELECT CURRENT_ROLE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ROLE_TYPE()", + "formatted": "SELECT CURRENT_ROLE_TYPE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAY(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAY(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAYOFMONTH(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())", + "formatted": "SELECT DAYOFYEAR(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTH(CURRENT_TIMESTAMP())", + "formatted": "SELECT MONTH(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT QUARTER(CURRENT_TIMESTAMP())", + "formatted": "SELECT QUARTER(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT WEEK(CURRENT_TIMESTAMP())", + "formatted": "SELECT WEEK(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT WEEKISO(CURRENT_TIMESTAMP())", + "formatted": "SELECT WEEKISO(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAR(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAR(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAROFWEEK(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())", + "formatted": "SELECT YEAROFWEEKISO(CURRENT_TIMESTAMP())" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM(amount) FROM mytable GROUP BY ALL", + "formatted": "SELECT SUM(amount)\nFROM mytable\nGROUP BY ALL" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV(x)", + "formatted": "SELECT STDDEV(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_POP(x)", + "formatted": "SELECT STDDEV_POP(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV_POP(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_SAMP(x)", + "formatted": "SELECT STDDEV_SAMP(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)", + "formatted": "SELECT STDDEV_SAMP(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT KURTOSIS(x)", + "formatted": "SELECT KURTOSIS(x)" + }, + { + "dialect": "snowflake", + "sql": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)", + "formatted": "SELECT KURTOSIS(x) OVER (PARTITION BY 1)" + }, + { + "dialect": "snowflake", + "sql": "WITH x AS (SELECT 1 AS foo) SELECT foo FROM IDENTIFIER('x')", + "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT foo\nFROM IDENTIFIER('x')" + }, + { + "dialect": "snowflake", + "sql": "WITH x AS (SELECT 1 AS foo) SELECT IDENTIFIER('foo') FROM x", + "formatted": "WITH x AS (\n SELECT 1 AS foo\n)\n\nSELECT IDENTIFIER('foo')\nFROM x" + }, + { + "dialect": "snowflake", + "sql": "SELECT IDENTIFIER($my_function_name)()", + "formatted": "SELECT IDENTIFIER($my_function_name)()" + }, + { + "dialect": "snowflake", + "sql": "SELECT IDENTIFIER('speed_of_light')()", + "formatted": "SELECT IDENTIFIER('speed_of_light')()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'", + "formatted": "SELECT CAST('2021-01-01' AS DATE) + INTERVAL '1 DAY'" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(*)", + "formatted": "SELECT HLL(*)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(a)", + "formatted": "SELECT HLL(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(DISTINCT t.a)", + "formatted": "SELECT HLL(DISTINCT t.a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(a, b, c)", + "formatted": "SELECT HLL(a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT HLL(DISTINCT a, b, c)", + "formatted": "SELECT HLL(DISTINCT a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_LIKE(a, b, c)", + "formatted": "SELECT REGEXP_LIKE(a, b, c)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)", + "formatted": "SELECT CONVERT_TIMEZONE('UTC', 'America/Los_Angeles', col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_ORGANIZATION_NAME()", + "formatted": "SELECT CURRENT_ORGANIZATION_NAME()" + }, + { + "dialect": "snowflake", + "sql": "SELECT MATCH_CONDITION", + "formatted": "SELECT MATCH_CONDITION" + }, + { + "dialect": "snowflake", + "sql": "SELECT OBJECT_AGG(key, value) FROM tbl", + "formatted": "SELECT OBJECT_AGG(key, value)\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))", + "formatted": "SELECT TO_CHAR(CAST('12:05:05' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), '')) FROM t", + "formatted": "SELECT TRIM(COALESCE(TO_CHAR(CAST(c AS TIME)), ''))\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')", + "formatted": "SELECT GET_PATH(PARSE_JSON(foo), 'bar')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_IP('192.168.1.1', 'INET')", + "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)", + "formatted": "SELECT PARSE_IP('192.168.1.1', 'INET', 0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(foo, 'bar')", + "formatted": "SELECT GET_PATH(foo, 'bar')" + }, + { + "dialect": "snowflake", + "sql": "SELECT a, exclude, b FROM xxx", + "formatted": "SELECT\n a,\n exclude,\n b\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_SORT(x, TRUE, FALSE)", + "formatted": "SELECT ARRAY_SORT(x, TRUE, FALSE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BOOLXOR_AGG(col) FROM tbl", + "formatted": "SELECT BOOLXOR_AGG(col)\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)", + "formatted": "SELECT PERCENTILE_DISC(0.9) WITHIN GROUP (ORDER BY col) OVER (PARTITION BY category)" + }, + { + "dialect": "snowflake", + "sql": "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + "formatted": "SELECT\n AI_CLASSIFY(\n 'text',\n ['travel', 'cooking'],\n OBJECT_CONSTRUCT('output_mode', 'multi')\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table", + "formatted": "SELECT *\nFROM table AT (TIMESTAMP => '2024-07-24') UNPIVOT(a FOR b IN (c)) AS pivot_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN ('2023_Q1', '2023_Q2', '2023_Q3', '2023_Q4', '2024_Q1') DEFAULT ON NULL (0))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT quarter FROM ad_campaign_types_by_quarter WHERE television = TRUE ORDER BY quarter)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (SELECT DISTINCT\n quarter\nFROM ad_campaign_types_by_quarter\nWHERE television = TRUE\nORDER BY\n quarter))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY ORDER BY quarter))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY)) ORDER BY empid", + "formatted": "SELECT *\nFROM quarterly_sales PIVOT(SUM(amount) FOR quarter IN (ANY))\nORDER BY\n empid" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM DATA AS DATA_L ASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL) ON DATA_L.ID = DATA_R.ID", + "formatted": "SELECT *\nFROM DATA AS DATA_L\nASOF JOIN DATA AS DATA_R MATCH_CONDITION (DATA_L.VAL > DATA_R.VAL)\n ON DATA_L.ID = DATA_R.ID" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')", + "formatted": "SELECT TO_TIMESTAMP('2025-01-16T14:45:30.123+0500', 'yyyy-mm-DDThh24:mi:ss.ff9tzhtzm')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * REPLACE (CAST(col AS TEXT) AS scol) FROM t", + "formatted": "SELECT * REPLACE (CAST(col AS TEXT) AS scol)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 put", + "formatted": "SELECT 1 put", + "previously_formatted": "SELECT\n 1 AS put" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 get", + "formatted": "SELECT 1 get", + "previously_formatted": "SELECT\n 1 AS get" + }, + { + "dialect": "snowflake", + "sql": "WITH t (SELECT 1 AS c) SELECT c FROM t", + "formatted": "WITH t (\n SELECT 1 AS c\n)\nSELECT c\nFROM t", + "previously_formatted": "WITH t AS (\n SELECT\n 1 AS c\n)\nSELECT\n c\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM s WHERE c NOT IN (1, 2, 3)", + "formatted": "SELECT *\nFROM s\nWHERE c NOT IN (1, 2, 3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM s WHERE c NOT IN (SELECT * FROM t)", + "formatted": "SELECT *\nFROM s\nWHERE\n c NOT IN (\n SELECT *\n FROM t\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM t1 INNER JOIN t2 USING (t1.col)", + "formatted": "SELECT *\nFROM t1\nINNER JOIN t2\n USING (t1.col)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a:from::STRING, a:from || ' test' ", + "formatted": "SELECT\n a:from::STRING,\n a:from || ' test'" + }, + { + "dialect": "snowflake", + "sql": "SELECT a:select", + "formatted": "SELECT a:select" + }, + { + "dialect": "snowflake", + "sql": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')", + "formatted": "SELECT GET_PATH(PARSE_JSON('{\"y\": [{\"z\": 1}]}'), 'y[0]:z')" + }, + { + "dialect": "snowflake", + "sql": "SELECT p FROM t WHERE p:val NOT IN ('2')", + "formatted": "SELECT p\nFROM t\nWHERE p:val NOT IN ('2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'", + "formatted": "SELECT PARSE_JSON('{\"x\": \"hello\"}'):x LIKE 'hello'" + }, + { + "dialect": "snowflake", + "sql": "SELECT data:x LIKE 'hello' FROM some_table", + "formatted": "SELECT data:x LIKE 'hello'\nFROM some_table" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })", + "formatted": "SELECT SUM({ fn CONVERT(123, SQL_DOUBLE) })" + }, + { + "dialect": "snowflake", + "sql": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })", + "formatted": "SELECT SUM({ fn CONVERT(123, SQL_VARCHAR) })" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMPFROMPARTS(d, t)", + "formatted": "SELECT TIMESTAMPFROMPARTS(d, t)" + }, + { + "dialect": "snowflake", + "sql": "SELECT v:attr[0].name FROM vartab", + "formatted": "SELECT v:attr[0].name\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT v:\"fruit\" FROM vartab", + "formatted": "SELECT v:\"fruit\"\nFROM vartab" + }, + { + "dialect": "snowflake", + "sql": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR", + "formatted": "SELECT PARSE_JSON('{\"food\":{\"fruit\":\"banana\"}}'):food.fruit::VARCHAR" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM t, UNNEST(x) WITH ORDINALITY", + "formatted": "SELECT *\nFROM t, UNNEST(x) WITH ORDINALITY" + }, + { + "dialect": "snowflake", + "sql": "SELECT state, city, SUM(retail_price * quantity) AS gross_revenue FROM sales GROUP BY ALL", + "formatted": "SELECT\n state,\n city,\n SUM(retail_price * quantity) AS gross_revenue\nFROM sales\nGROUP BY ALL" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM foo window", + "formatted": "SELECT *\nFROM foo window", + "previously_formatted": "SELECT\n *\nFROM foo AS window" + }, + { + "dialect": "snowflake", + "sql": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i') FROM log_source", + "formatted": "SELECT RLIKE(a, $$regular expression with \\ characters: \\d{2}-\\d{3}-\\d{4}$$, 'i')\nFROM log_source" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a ' \\ \\t \\x21 z $ $$", + "formatted": "SELECT $$a ' \\ \\t \\x21 z $ $$" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a\\$b$$", + "formatted": "SELECT $$a\\$b$$" + }, + { + "dialect": "snowflake", + "sql": "SELECT $$a\\$$", + "formatted": "SELECT $$a\\$$" + }, + { + "dialect": "snowflake", + "sql": "SELECT {'test': 'best'}::VARIANT", + "formatted": "SELECT {'test': 'best'}::VARIANT" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn DAYNAME('2022-5-13')}", + "formatted": "SELECT {fn\n DAYNAME('2022-5-13') }" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn LOG(5)}", + "formatted": "SELECT {fn\n LOG(5) }" + }, + { + "dialect": "snowflake", + "sql": "SELECT {fn CEILING(5.3)}", + "formatted": "SELECT {fn\n CEILING(5.3) }" + }, + { + "dialect": "snowflake", + "sql": "SELECT CEIL(3.14)", + "formatted": "SELECT CEIL(3.14)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CEIL(3.14, 1)", + "formatted": "SELECT CEIL(3.14, 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)", + "formatted": "SELECT DAYOFWEEK('2016-01-02T23:39:20.123-07:00'::TIMESTAMP)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx WHERE col ilike '%Don''t%'", + "formatted": "SELECT *\nFROM xxx\nWHERE col ILIKE '%Don''t%'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * EXCLUDE a, b FROM xxx", + "formatted": "SELECT\n * EXCLUDE a,\n b\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT * RENAME a AS b, c AS d FROM xxx", + "formatted": "SELECT\n * RENAME a AS b,\n c AS d\nFROM xxx" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz,", + "formatted": "SELECT *\nFROM xxx, yyy, zzz," + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz, WHERE foo = bar", + "formatted": "SELECT *\nFROM xxx, yyy, zzz,\nWHERE foo = bar" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM xxx, yyy, zzz", + "formatted": "SELECT *\nFROM xxx, yyy, zzz" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)", + "formatted": "SELECT TIMESTAMPNTZFROMPARTS(2013, 4, 5, 12, 00, 00)" + }, + { + "dialect": "snowflake", + "sql": "SELECT NTH_VALUE(a, 2) FROM t", + "formatted": "SELECT NTH_VALUE(a, 2)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]", + "formatted": "SELECT ARRAY_CONSTRUCT('foo')::VARIANT[0]" + }, + { + "dialect": "snowflake", + "sql": "SELECT UUID_STRING(), UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')", + "formatted": "SELECT\n UUID_STRING(),\n UUID_STRING('fe971b24-9572-4005-b22f-351e9c09274d', 'foo')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITNOT(a)", + "formatted": "SELECT BITNOT(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_NOT(a)", + "formatted": "SELECT BIT_NOT(a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITAND(a, b)", + "formatted": "SELECT BITAND(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITAND(a, b, 'LEFT')", + "formatted": "SELECT BITAND(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_AND(a, b)", + "formatted": "SELECT BIT_AND(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_AND(a, b, 'LEFT')", + "formatted": "SELECT BIT_AND(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITOR(a, b)", + "formatted": "SELECT BITOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITOR(a, b, 'LEFT')", + "formatted": "SELECT BITOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_OR(a, b)", + "formatted": "SELECT BIT_OR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_OR(a, b, 'RIGHT')", + "formatted": "SELECT BIT_OR(a, b, 'RIGHT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITXOR(a, b)", + "formatted": "SELECT BITXOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BITXOR(a, b, 'LEFT')", + "formatted": "SELECT BITXOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_XOR(a, b)", + "formatted": "SELECT BIT_XOR(a, b)" + }, + { + "dialect": "snowflake", + "sql": "SELECT BIT_XOR(a, b, 'LEFT')", + "formatted": "SELECT BIT_XOR(a, b, 'LEFT')" + }, + { + "dialect": "snowflake", + "sql": "SELECT HOUR(CAST('08:50:57' AS TIME))", + "formatted": "SELECT HOUR(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MINUTE(CAST('08:50:57' AS TIME))", + "formatted": "SELECT MINUTE(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT SECOND(CAST('08:50:57' AS TIME))", + "formatted": "SELECT SECOND(CAST('08:50:57' AS TIME))" + }, + { + "dialect": "snowflake", + "sql": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))", + "formatted": "SELECT HOUR(CAST('2024-05-09 08:50:57' AS TIMESTAMP))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))", + "formatted": "SELECT MONTHNAME(CAST('2024-05-09' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')", + "formatted": "SELECT PREVIOUS_DAY(CAST('2024-05-09' AS DATE), 'MONDAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_FROM_PARTS(14, 30, 45)", + "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)", + "formatted": "SELECT TIME_FROM_PARTS(14, 30, 45, 123)" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))", + "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-15' AS DATE), CAST('2019-02-15' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))", + "formatted": "SELECT MONTHS_BETWEEN(CAST('2019-03-01 02:00:00' AS TIMESTAMP), CAST('2019-02-15 01:00:00' AS TIMESTAMP))" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')", + "formatted": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 15, 'MINUTE')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')", + "formatted": "SELECT TIME_SLICE(CAST('2024-05-09' AS DATE), 1, 'DAY')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + "formatted": "SELECT\n TIME_SLICE(\n CAST('2024-05-09 08:50:57.891' AS TIMESTAMP),\n 1,\n 'HOUR',\n 'start'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT * EXCLUDE foo RENAME bar AS baz FROM tbl", + "formatted": "SELECT * EXCLUDE foo RENAME bar AS baz\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern')", + "formatted": "SELECT LIKE(col, 'pattern')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern')", + "formatted": "SELECT ILIKE(col, 'pattern')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern', '\\\\')", + "formatted": "SELECT LIKE(col, 'pattern', '\\\\')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern', '\\\\')", + "formatted": "SELECT ILIKE(col, 'pattern', '\\\\')" + }, + { + "dialect": "snowflake", + "sql": "SELECT LIKE(col, 'pattern', '!')", + "formatted": "SELECT LIKE(col, 'pattern', '!')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ILIKE(col, 'pattern', '!')", + "formatted": "SELECT ILIKE(col, 'pattern', '!')" + }, + { + "dialect": "snowflake", + "sql": "SELECT BASE64_ENCODE('Hello World')", + "formatted": "SELECT BASE64_ENCODE('Hello World')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')", + "formatted": "SELECT TRY_HEX_DECODE_BINARY('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')", + "formatted": "SELECT TRY_HEX_DECODE_STRING('48656C6C6F')" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_CONTAINS(1, [1])", + "formatted": "SELECT ARRAY_CONTAINS(1, [1])" + }, + { + "dialect": "snowflake", + "sql": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])", + "formatted": "SELECT ARRAY_DISTINCT(['A', 'B', 'A'])" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_DATABASE()", + "formatted": "SELECT CURRENT_DATABASE()" + }, + { + "dialect": "snowflake", + "sql": "SELECT CURRENT_SCHEMA()", + "formatted": "SELECT CURRENT_SCHEMA()" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"mystage\"", + "formatted": "SELECT *\nFROM @\"mystage\"" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"myschema\".\"mystage\"/file.gz", + "formatted": "SELECT *\nFROM @\"myschema\".\"mystage\"/file.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @\"my_DB\".\"schEMA1\".mystage/file.gz", + "formatted": "SELECT *\nFROM @\"my_DB\".\"schEMA1\".mystage/file.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT metadata$filename FROM @s1/", + "formatted": "SELECT metadata$filename\nFROM @s1/" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @~", + "formatted": "SELECT *\nFROM @~" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @~/some/path/to/file.csv", + "formatted": "SELECT *\nFROM @~/some/path/to/file.csv" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @mystage", + "formatted": "SELECT *\nFROM @mystage" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM '@mystage'", + "formatted": "SELECT *\nFROM '@mystage'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @namespace.mystage/path/to/file.json.gz", + "formatted": "SELECT *\nFROM @namespace.mystage/path/to/file.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @namespace.%table_name/path/to/file.json.gz", + "formatted": "SELECT *\nFROM @namespace.%table_name/path/to/file.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1, $2, metadata$filename FROM @mystage (PATTERN => '.*data-100.*')", + "formatted": "SELECT\n $1,\n $2,\n metadata$filename\nFROM @mystage (PATTERN => '.*data-100.*')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv')", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT a FROM @foo)", + "formatted": "SELECT *\nFROM (\n SELECT a\n FROM @foo\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT * FROM '@external/location' (FILE_FORMAT => 'path.to.csv'))", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla", + "formatted": "SELECT *\nFROM @foo/bar (FILE_FORMAT => ds_sandbox.test.my_csv_format, PATTERN => 'test') AS bla" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$1, t.$2 FROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t", + "formatted": "SELECT\n t.$1,\n t.$2\nFROM @mystage1 (FILE_FORMAT => 'myformat', PATTERN => '.*data.*[.]csv.gz') AS t" + }, + { + "dialect": "snowflake", + "sql": "SELECT parse_json($1):a.b FROM @mystage2/data1.json.gz", + "formatted": "SELECT parse_json($1):a.b\nFROM @mystage2/data1.json.gz" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @mystage t (c1)", + "formatted": "SELECT *\nFROM @mystage t(c1)", + "previously_formatted": "SELECT\n *\nFROM @mystage AS t(c1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla", + "formatted": "SELECT *\nFROM @foo/bar (PATTERN => 'test', FILE_FORMAT => ds_sandbox.test.my_csv_format) AS bla" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM @test.public.thing/location/somefile.csv( FILE_FORMAT => 'fmt' )", + "formatted": "SELECT *\nFROM @test.public.thing/location/somefile.csv (FILE_FORMAT => 'fmt')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE BERNOULLI (20.3)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE BERNOULLI (20.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE SYSTEM (3) SEED (82)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE SYSTEM (3) SEED (82)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a FROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)", + "formatted": "SELECT a\nFROM test PIVOT(SUM(x) FOR y IN ('z', 'q')) AS x TABLESAMPLE BERNOULLI (0.1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT i, j FROM table1 AS t1 INNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50) WHERE t2.j = t1.i", + "formatted": "SELECT\n i,\n j\nFROM table1 AS t1\nINNER JOIN table2 AS t2 TABLESAMPLE BERNOULLI (50)\nWHERE t2.j = t1.i" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (SELECT * FROM t1 JOIN t2 ON t1.a = t2.c) TABLESAMPLE BERNOULLI (1)", + "formatted": "SELECT *\nFROM (\n SELECT *\n FROM t1\n JOIN t2\n ON t1.a = t2.c\n) TABLESAMPLE BERNOULLI (1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE (10 ROWS)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE (10 ROWS)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable TABLESAMPLE (100)", + "formatted": "SELECT *\nFROM testtable TABLESAMPLE (100)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable SAMPLE (10)", + "formatted": "SELECT *\nFROM testtable SAMPLE (10)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM testtable SAMPLE ROW (0)", + "formatted": "SELECT *\nFROM testtable SAMPLE ROW (0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT a FROM test SAMPLE BLOCK (0.5) SEED (42)", + "formatted": "SELECT a\nFROM test SAMPLE BLOCK (0.5) SEED (42)" + }, + { + "dialect": "snowflake", + "sql": "SELECT user_id, value FROM table_name SAMPLE BERNOULLI ($s) SEED (0)", + "formatted": "SELECT\n user_id,\n value\nFROM table_name SAMPLE BERNOULLI ($s) SEED (0)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST('12:00:00' AS TIME)", + "formatted": "SELECT CAST('12:00:00' AS TIME)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(month, a)", + "formatted": "SELECT DATE_PART(month, a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIME(x) FROM t", + "formatted": "SELECT TO_TIME(x)\nFROM t" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'", + "formatted": "SELECT TO_DATE('2019-02-28') + INTERVAL '1 day, 1 year'" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(a AS VARIANT)", + "formatted": "SELECT CAST(a AS VARIANT)" + }, + { + "dialect": "snowflake", + "sql": "SELECT CAST(a AS ARRAY)", + "formatted": "SELECT CAST(a AS ARRAY)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (STATEMENT => $query_id_var)", + "formatted": "SELECT *\nFROM my_table AT (STATEMENT => $query_id_var)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5)", + "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (STATEMENT => $query_id_var)", + "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => $query_id_var)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (OFFSET => -60 * 5)", + "formatted": "SELECT *\nFROM my_table BEFORE (OFFSET => -60 * 5)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => TO_TIMESTAMP(1432669154242, 3))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (OFFSET => -60 * 5) AS T WHERE T.flag = 'valid'", + "formatted": "SELECT *\nFROM my_table AT (OFFSET => -60 * 5) AS T\nWHERE T.flag = 'valid'" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", + "formatted": "SELECT *\nFROM my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')", + "formatted": "SELECT *\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table AT(TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)", + "formatted": "SELECT *\nFROM my_table AT (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", + "formatted": "SELECT *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz);", + "previously_formatted": "SELECT\n *\nFROM my_table BEFORE (TIMESTAMP => 'Fri, 01 May 2015 16:20:00 -0700'::timestamp_tz)" + }, + { + "dialect": "snowflake", + "sql": "\n SELECT oldt.* , newt.*\n FROM my_table BEFORE(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\n FULL OUTER JOIN my_table AT(STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\n WHERE oldt.id IS NULL OR newt.id IS NULL;\n ", + "formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL;", + "previously_formatted": "SELECT\n oldt.*,\n newt.*\nFROM my_table BEFORE (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS oldt\nFULL OUTER JOIN my_table AT (STATEMENT => '8e5d0ca9-005e-44e6-b858-a8f5b37c5726') AS newt\n ON oldt.id = newt.id\nWHERE\n oldt.id IS NULL\n OR newt.id IS NULL" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('MYTABLE')", + "formatted": "SELECT *\nFROM TABLE('MYTABLE')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($MYVAR)", + "formatted": "SELECT *\nFROM TABLE($MYVAR)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(?)", + "formatted": "SELECT *\nFROM TABLE(?)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(:BINDING)", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($MYVAR) WHERE COL1 = 10", + "formatted": "SELECT *\nFROM TABLE($MYVAR)\nWHERE COL1 = 10" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') AS f", + "formatted": "SELECT *\nFROM TABLE('t1') AS f" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM (TABLE('t1') CROSS JOIN TABLE('t2'))", + "formatted": "SELECT *\nFROM (TABLE('t1')\nCROSS JOIN TABLE('t2'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1'), LATERAL (SELECT * FROM t2)", + "formatted": "SELECT *\nFROM TABLE('t1'), LATERAL (\n SELECT *\n FROM t2\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') UNION ALL SELECT * FROM TABLE('t2')", + "formatted": "SELECT *\nFROM TABLE('t1')\nUNION ALL\nSELECT *\nFROM TABLE('t2')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)", + "formatted": "SELECT *\nFROM TABLE('t1') TABLESAMPLE BERNOULLI (20.3)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')", + "formatted": "SELECT *\nFROM TABLE('MYDB.\"MYSCHEMA\".\"MYTABLE\"')" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)", + "formatted": "SELECT *\nFROM TABLE($$MYDB. \"MYSCHEMA\".\"MYTABLE\"$$)" + }, + { + "dialect": "snowflake", + "sql": "SELECT value FROM TABLE(FLATTEN(input => SELECT PARSE_JSON('[1, 2]')))", + "formatted": "SELECT value\nFROM TABLE(FLATTEN(input => SELECT\n PARSE_JSON('[1, 2]')))" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH((play, line), 'dream')", + "formatted": "SELECT SEARCH((play, line), 'dream')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", + "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", + "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n ANALYZER => 'UNICODE_ANALYZER',\n SEARCH_MODE => 'OR'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king')", + "formatted": "SELECT SEARCH(line, 'king')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')", + "formatted": "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')", + "formatted": "SELECT SEARCH(character, 'king queen', SEARCH_MODE => 'AND')" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + "formatted": "SELECT\n SEARCH(\n line,\n 'king',\n SEARCH_MODE => 'AND',\n ANALYZER => 'PATTERN_ANALYZER'\n )" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l ')", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l ')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1)", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')", + "formatted": "SELECT REGEXP_COUNT('hello world', 'l', 1, 'i')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_INSTR('abc', 'a')", + "formatted": "SELECT REGEXP_INSTR('abc', 'a')" + }, + { + "dialect": "snowflake", + "sql": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')", + "formatted": "SELECT REGEXP_INSTR('abc', 'a', 1, 1, 0, 'i')" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_CAST(x AS DOUBLE)", + "formatted": "SELECT TRY_CAST(x AS DOUBLE)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_CAST(FOO() AS TEXT)", + "formatted": "SELECT TRY_CAST(FOO() AS TEXT)" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1", + "formatted": "SELECT $1" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1.elem", + "formatted": "SELECT $1.elem" + }, + { + "dialect": "snowflake", + "sql": "SELECT $1:a.b", + "formatted": "SELECT $1:a.b" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$23:a.b", + "formatted": "SELECT t.$23:a.b" + }, + { + "dialect": "snowflake", + "sql": "SELECT t.$17:a[0].b[0].c", + "formatted": "SELECT t.$17:a[0].b[0].c" + }, + { + "dialect": "snowflake", + "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)", + "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) AT (STREAM => 's1') END (TIMESTAMP => $ts2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT C1 FROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)", + "formatted": "SELECT C1\nFROM t1 CHANGES (INFORMATION => APPEND_ONLY) BEFORE (STATEMENT => 'STMT_ID') END (TIMESTAMP => $ts2)" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))", + "formatted": "SELECT 1\nFROM some_table CHANGES (INFORMATION => APPEND_ONLY) AT (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 00:00:00+00:00')) END (TIMESTAMP => TO_TIMESTAMP_TZ('2024-07-01 14:28:59.999999+00:00'))" + }, + { + "dialect": "snowflake", + "sql": "SELECT :1", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT :1, :2", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT :1 + :2", + "formatted": null + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM SEMANTIC_VIEW(foo METRICS a.b, a.c DIMENSIONS a.b, a.c WHERE a.b > '1995-01-01')", + "formatted": "SELECT *\nFROM SEMANTIC_VIEW(\n foo\n METRICS a.b, a.c\n DIMENSIONS a.b, a.c\n WHERE a.b > '1995-01-01'\n)" + }, + { + "dialect": "snowflake", + "sql": "SELECT col1, col2, metric1 FROM SEMANTIC_VIEW(mydb.myschema.my_semantic_view METRICS metric1 DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2) ORDER BY col1, col2 DESC", + "formatted": "SELECT\n col1,\n col2,\n metric1\nFROM SEMANTIC_VIEW(\n mydb.myschema.my_semantic_view\n METRICS metric1\n DIMENSIONS col1, DATE_TRUNC('MONTH', timestamp_col) AS col2\n)\nORDER BY\n col1,\n col2 DESC" + }, + { + "dialect": "snowflake", + "sql": "SELECT model!mladmin", + "formatted": "SELECT model!mladmin" + }, + { + "dialect": "snowflake", + "sql": "SELECT model!PREDICT(1)", + "formatted": "SELECT model!PREDICT(1)" + }, + { + "dialect": "snowflake", + "sql": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p FROM tbl", + "formatted": "SELECT m!PREDICT(INPUT_DATA => {*}) AS p\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p FROM tbl", + "formatted": "SELECT m!PREDICT(INPUT_DATA => {tbl.*}) AS p\nFROM tbl" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(INPUT_DATA => TABLE(view_with_data_to_analyze), TIMESTAMP_COLNAME => 'date', TARGET_COLNAME => 'sales', CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)))", + "formatted": "SELECT *\nFROM TABLE(model_trained_with_labeled_data!DETECT_ANOMALIES(\n INPUT_DATA => TABLE(view_with_data_to_analyze),\n TIMESTAMP_COLNAME => 'date',\n TARGET_COLNAME => 'sales',\n CONFIG_OBJECT => OBJECT_CONSTRUCT('prediction_interval', 0.99)\n))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(TIMELIMIT => 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(TIMELIMIT => 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(ROWCOUNT => 10, TIMELIMIT => 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(10))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10))" + }, + { + "dialect": "snowflake", + "sql": "SELECT 1 FROM TABLE(GENERATOR(10, 5))", + "formatted": "SELECT 1\nFROM TABLE(GENERATOR(10, 5))" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a CROSS DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nCROSS DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a INNER DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nINNER DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT * FROM a NATURAL INNER DIRECTED JOIN b USING (id)", + "formatted": "SELECT *\nFROM a\nNATURAL INNER DIRECTED JOIN b\n USING (id)" + }, + { + "dialect": "snowflake", + "sql": "SELECT TRY_PARSE_JSON('{\"x: 1}')", + "formatted": "SELECT TRY_PARSE_JSON('{\"x: 1}')" + }, + { + "dialect": "snowflake", + "sql": "SELECT number", + "formatted": "SELECT number" + }, + { + "dialect": "snowflake", + "sql": "SELECT TO_TIMESTAMP(123.4)", + "formatted": "SELECT TO_TIMESTAMP(123.4)" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))", + "formatted": "SELECT DATE_PART(year FROM CAST('2024-04-08' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", + "formatted": "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATE_PART(day FROM a)", + "formatted": "SELECT DATE_PART(day FROM a)" + }, + { + "dialect": "snowflake", + "sql": "SELECT SEARCH_IP(col, '192.168.0.0')", + "formatted": "SELECT SEARCH_IP(col, '192.168.0.0')" + }, + { + "dialect": "snowflake", + "sql": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE))) FROM (SELECT 'month' AS m) AS t", + "formatted": "SELECT DATEADD(DAY, -7, DATEADD(t.m, 1, CAST('2023-01-03' AS DATE)))\nFROM (\n SELECT 'month' AS m\n) AS t" + }, + { + "dialect": "snowflake", + "sql": "SELECT FILE_URL FROM DIRECTORY(@mystage) WHERE SIZE > 100000", + "formatted": "SELECT FILE_URL\nFROM DIRECTORY(@mystage)\nWHERE SIZE > 100000" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM a..b", + "formatted": "SELECT *\nFROM a..b" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP (SELECT 1) * FROM t", + "formatted": "SELECT TOP (\n SELECT 1\n)\n *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "SELECT ATN2(x, y)", + "formatted": "SELECT ATN2(x, y)" + }, + { + "dialect": "tsql", + "sql": "SELECT EXP(1)", + "formatted": "SELECT EXP(1)" + }, + { + "dialect": "tsql", + "sql": "SELECT SYSDATETIMEOFFSET()", + "formatted": "SELECT SYSDATETIMEOFFSET()" + }, + { + "dialect": "tsql", + "sql": "SELECT COMPRESS('Hello World')", + "formatted": "SELECT COMPRESS('Hello World')" + }, + { + "dialect": "tsql", + "sql": "SELECT TRIM(' test ') AS Result", + "formatted": "SELECT TRIM(' test ') AS Result" + }, + { + "dialect": "tsql", + "sql": "SELECT TRIM('.,! ' FROM ' # test .') AS Result", + "formatted": "SELECT TRIM('.,! ' FROM ' # test .') AS Result" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (20 ROWS)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (20 ROWS)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)", + "formatted": "SELECT *\nFROM t TABLESAMPLE (10 PERCENT) REPEATABLE (123)" + }, + { + "dialect": "tsql", + "sql": "SELECT CONCAT(column1, column2)", + "formatted": "SELECT CONCAT(column1, column2)" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test# FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test#\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test@ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test@\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test$ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test$\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TestSpecialChar.Test_ FROM TestSpecialChar", + "formatted": "SELECT TestSpecialChar.Test_\nFROM TestSpecialChar" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP (2 + 1) 1", + "formatted": "SELECT TOP (2 + 1) 1" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WHERE NOT c", + "formatted": "SELECT *\nFROM t\nWHERE NOT c" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 s.RECORDID, n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE FROM source_table.dbo.source_data AS s(nolock) CROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)", + "formatted": "SELECT TOP 10\n s.RECORDID,\n n.c.VALUE('(/*:FORM_ROOT/*:SOME_TAG)[1]', 'float') AS SOME_TAG_VALUE\nFROM source_table.dbo.source_data AS s(nolock)\nCROSS APPLY FormContent.nodes('/*:FORM_ROOT') AS N(C)" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT TOP 10 a\nFROM t1\nUNION ALL\nSELECT TOP 10 a\nFROM t2\nORDER BY a DESC" + }, + { + "dialect": "tsql", + "sql": "WITH t1 AS (SELECT 1 AS a), t2 AS (SELECT 1 AS a) SELECT COUNT(*) FROM (SELECT TOP 10 a FROM t1 UNION ALL SELECT TOP 10 a FROM t2 ORDER BY a DESC) AS t", + "formatted": "WITH t1 AS (\n SELECT 1 AS a\n),\n\nt2 AS (\n SELECT 1 AS a\n)\n\nSELECT COUNT(*)\nFROM (\n SELECT TOP 10 a\n FROM t1\n UNION ALL\n SELECT TOP 10 a\n FROM t2\nORDER BY a DESC\n) AS t" + }, + { + "dialect": "tsql", + "sql": "SELECT 1 AS \"[x]\"", + "formatted": "SELECT 1 AS \"[x]\"" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WITH (TABLOCK, INDEX(myindex))", + "formatted": "SELECT *\nFROM t WITH (TABLOCK, INDEX(myindex))" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t WITH (NOWAIT)", + "formatted": "SELECT *\nFROM t WITH (NOWAIT)" + }, + { + "dialect": "tsql", + "sql": "SELECT CASE WHEN a > 1 THEN b END", + "formatted": "SELECT CASE WHEN a > 1 THEN b END" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM taxi ORDER BY 1 OFFSET 0 ROWS FETCH NEXT 3 ROWS ONLY", + "formatted": "SELECT *\nFROM taxi\nORDER BY\n 1\nOFFSET 0 ROWS\nFETCH NEXT 3 ROWS ONLY" + }, + { + "dialect": "tsql", + "sql": "SELECT Employee_ID, Department_ID FROM @MyTableVar", + "formatted": "SELECT\n Employee_ID,\n Department_ID\nFROM @MyTableVar" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM #foo", + "formatted": "SELECT *\nFROM #foo" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM ##foo", + "formatted": "SELECT *\nFROM ##foo" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1", + "formatted": "SELECT a = 1" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1 UNION ALL SELECT a = b", + "formatted": "SELECT a = 1\nUNION ALL\nSELECT a = b" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM @MyTableVar AS m JOIN Employee ON m.EmployeeID = Employee.EmployeeID", + "formatted": "SELECT x\nFROM @MyTableVar AS m\nJOIN Employee\n ON m.EmployeeID = Employee.EmployeeID" + }, + { + "dialect": "tsql", + "sql": "SELECT DISTINCT DepartmentName, PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont FROM dbo.DimEmployee", + "formatted": "SELECT DISTINCT\n DepartmentName,\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY BaseRate) OVER (PARTITION BY DepartmentName) AS MedianCont\nFROM dbo.DimEmployee" + }, + { + "dialect": "tsql", + "sql": "SELECT \"x\".\"y\" FROM foo", + "formatted": "SELECT \"x\".\"y\"\nFROM foo" + }, + { + "dialect": "tsql", + "sql": "SELECT val FROM (VALUES ((TRUE), (FALSE), (NULL))) AS t(val)", + "formatted": "SELECT val\nFROM (\n VALUES\n ((TRUE\n), (FALSE\n), (\n NULL\n))\n) AS t(val)" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR XML PATH, BINARY BASE64, ELEMENTS XSINIL", + "formatted": "SELECT *\nFROM t\nFOR XML\n PATH,\n BINARY BASE64,\n ELEMENTS XSINIL" + }, + { + "dialect": "tsql", + "sql": "SELECT (SELECT TOP 5 a, b FROM t ORDER BY b DESC FOR JSON PATH) AS j FROM x", + "formatted": "SELECT\n (\n SELECT TOP 5\n a,\n b\n FROM t\n ORDER BY\n b DESC\n FOR JSON\n PATH\n ) AS j\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT j FROM (SELECT a AS j FROM t FOR JSON PATH) AS x", + "formatted": "SELECT j\nFROM (\n SELECT a AS j\n FROM t\n FOR JSON\n PATH\n) AS x" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR JSON PATH, ROOT('Root'), INCLUDE_NULL_VALUES", + "formatted": "SELECT *\nFROM t\nFOR JSON\n PATH,\n ROOT('Root'),\n INCLUDE_NULL_VALUES" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM t FOR BROWSE", + "formatted": "SELECT *\nFROM t FOR BROWSE" + }, + { + "dialect": "tsql", + "sql": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)", + "formatted": "SELECT CAST(SUBSTRING('ABCD~1234', CHARINDEX('~', 'ABCD~1234') + 1, LEN('ABCD~1234')) AS BIGINT)" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEPART(\"dd\", x)", + "formatted": "SELECT DATEPART(\"dd\", x)" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEADD(YEAR, 1, '2017/08/25')", + "formatted": "SELECT DATEADD(YEAR, 1, '2017/08/25')" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')", + "formatted": "SELECT DATEDIFF(HOUR, 1.5, '2021-01-01')" + }, + { + "dialect": "tsql", + "sql": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')", + "formatted": "SELECT DATEDIFF_BIG(HOUR, 1.5, '2021-01-01')" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT", + "formatted": "SELECT TOP 10 PERCENT" + }, + { + "dialect": "tsql", + "sql": "SELECT TOP 10 PERCENT WITH TIES", + "formatted": "SELECT TOP 10 PERCENT WITH TIES" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(foo, 'dddd', 'de-CH')", + "formatted": "SELECT FORMAT(foo, 'dddd', 'de-CH')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')", + "formatted": "SELECT FORMAT(EndOfDayRate, 'N', 'en-us')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')", + "formatted": "SELECT FORMAT('01-01-1991', 'd.mm.yyyy')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(12345, '###.###.###')", + "formatted": "SELECT FORMAT(12345, '###.###.###')" + }, + { + "dialect": "tsql", + "sql": "SELECT FORMAT(1234567, 'f')", + "formatted": "SELECT FORMAT(1234567, 'f')" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo'" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME AS OF 'foo' AS alias" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME FROM c TO d", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME FROM c TO d" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME BETWEEN c AND d" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME CONTAINED IN (c, d)" + }, + { + "dialect": "tsql", + "sql": "SELECT [x] FROM [a].[b] FOR SYSTEM_TIME ALL AS alias", + "formatted": "SELECT [x]\nFROM [a].[b] FOR SYSTEM_TIME ALL AS alias" + }, + { + "dialect": "tsql", + "sql": "SELECT x FROM a INNER LOOP JOIN b ON b.id = a.id", + "formatted": "SELECT x\nFROM a\nINNER LOOP JOIN b\n ON b.id = a.id" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM OPENJSON(@json)", + "formatted": "SELECT *\nFROM OPENJSON(@json)" + }, + { + "dialect": "tsql", + "sql": "WITH t AS (SELECT 1) SELECT * FROM t", + "formatted": "WITH t AS (\n SELECT 1\n)\n\nSELECT *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "WITH t AS (SELECT \"c\") SELECT * FROM t", + "formatted": "WITH t AS (\n SELECT \"c\"\n)\n\nSELECT *\nFROM t" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM (SELECT 1) AS subq", + "formatted": "SELECT *\nFROM (\n SELECT 1\n) AS subq" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM (SELECT \"c\") AS subq", + "formatted": "SELECT *\nFROM (\n SELECT \"c\"\n) AS subq" + }, + { + "dialect": "tsql", + "sql": "SELECT NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo), col", + "formatted": "SELECT\n NEXT VALUE FOR db.schema.sequence_name OVER (ORDER BY foo),\n col" + }, + { + "dialect": "tsql", + "sql": "SELECT COUNT(1) FROM x", + "formatted": "SELECT COUNT(1)\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT COUNT_BIG(1) FROM x", + "formatted": "SELECT COUNT_BIG(1)\nFROM x" + }, + { + "dialect": "tsql", + "sql": "SELECT 1; SELECT 2", + "formatted": "SELECT 1;\nSELECT 2" + }, + { + "dialect": "tsql", + "sql": "SELECT go", + "formatted": "SELECT go" + }, + { + "dialect": "tsql", + "sql": "SELECT 1 WHERE EXISTS(SELECT 1)", + "formatted": "SELECT 1\nWHERE\n EXISTS(\n SELECT 1\n )" + }, + { + "dialect": "tsql", + "sql": "SELECT a = 1", + "formatted": "SELECT a = 1" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM [#temp_table]", + "formatted": "SELECT *\nFROM [#temp_table]" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM [##temp_table]", + "formatted": "SELECT *\nFROM [##temp_table]" + }, + { + "dialect": "tsql", + "sql": "SELECT * FROM @x", + "formatted": "SELECT *\nFROM @x" + } +] diff --git a/crates/sqlquack-format/src/format/tests/helpers.rs b/crates/sqlquack-format/src/format/tests/helpers.rs new file mode 100644 index 000000000..a9595059c --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/helpers.rs @@ -0,0 +1,163 @@ +//! Helpers of the format tests. + +use serde_json::Value; +use sqlquack_syntax::{Dialect, TokenKind, tokenize}; + +use crate::format::constants::{DEPTH_FAILURE, PARSE_FAILURE, TYPE_DEPTH_FAILURE}; +use crate::format::main::format_sql::format_sql; +use crate::format::models::FormatOptions; +use crate::format::tests::test_types::{CorpusOutcome, EntryOutcome, FormatCorpusTestCase}; + +/// The line width the corpus is also wrapped at, as SQLClean's corpus test does. +const CORPUS_LINE_WIDTH: usize = 40; +/// The corpus dialect names SQLQuack tokenizes, with their dialects. +const DIALECTS: [(&str, Dialect); 3] = [ + ("snowflake", Dialect::Snowflake), + ("duckdb", Dialect::DuckDB), + ("postgres", Dialect::PostgreSQL), +]; + +fn dialect(name: &str) -> Dialect { + DIALECTS + .iter() + .find(|(known, _)| *known == name) + .map_or(Dialect::Snowflake, |(_, dialect)| *dialect) +} + +/// Format `sql`; `None` when the formatter refuses it. +fn format_once(sql: &str, dialect: Dialect, line_width: Option) -> Option { + let options = FormatOptions { + line_width, + ..FormatOptions::new(dialect) + }; + format_sql(sql, &options) + .ok() + .map(|formatted| formatted.sql) +} + +/// The significant token texts of `sql`. +fn token_texts(sql: &str, dialect: Dialect) -> Vec { + tokenize(sql, dialect) + .unwrap_or_default() + .iter() + .filter(|token| matches!(token.kind, TokenKind::Significant(_))) + .map(|token| token.text(sql).to_string()) + .collect() +} + +/// Whether `printed` has the tokens of `sql` except for words printed in upper case. +fn only_keyword_case_differs(sql: &str, printed: &str, dialect: Dialect) -> bool { + let (before, after) = (token_texts(sql, dialect), token_texts(printed, dialect)); + before.len() == after.len() + && before.iter().zip(&after).all(|(left, right)| { + left == right + || (left.eq_ignore_ascii_case(right) && *right == right.to_ascii_uppercase()) + }) +} + +/// Whether a layout of an earlier formatter release is relaid with its tokens kept. +fn relays_previous(previous: &str, dialect: Dialect) -> bool { + format_once(previous, dialect, None) + .is_some_and(|relaid| only_keyword_case_differs(previous, &relaid, dialect)) +} + +/// Whether formatting `stable` again leaves it as it is. +fn stays(stable: &str, dialect: Dialect, line_width: Option) -> bool { + format_once(stable, dialect, line_width).as_deref() == Some(stable) +} + +/// Format one corpus query and run every check SQLClean's corpus test runs on it. +fn check_entry(entry: &Value, test_case: &FormatCorpusTestCase) -> EntryOutcome { + let name = test_case.dialect; + let dialect = dialect(name); + let sql = entry["sql"].as_str().unwrap_or_default(); + let departure = test_case + .departures + .iter() + .find(|departure| departure.sql == sql); + let recorded = entry["formatted"].as_str(); + let expected: Option = departure + .map_or(recorded, |departure| departure.formatted) + .map(str::to_string); + let receipt = departure.map(|departure| (departure.reason, departure.receipt)); + let actual = format_once(sql, dialect, None); + let wrapped = format_once(sql, dialect, Some(CORPUS_LINE_WIDTH)); + let previous = entry["previously_formatted"].as_str(); + let checks = [ + ( + actual == expected, + format!( + "{name} layout differs (departure {receipt:?}):\n{sql}\n--- expected\n{}\n--- actual\n{}", + expected.as_deref().unwrap_or(""), + actual.as_deref().unwrap_or("") + ), + ), + ( + departure.is_none_or(|departure| departure.formatted != recorded), + format!("{name} departure matches SQLClean's layout: {sql}"), + ), + ( + only_keyword_case_differs(sql, actual.as_deref().unwrap_or(sql), dialect), + format!("{name} tokens changed beyond keyword case: {sql}"), + ), + ( + previous.is_none_or(|previous| relays_previous(previous, dialect)), + format!("{name} previous layout not relaid: {previous:?}"), + ), + ( + expected + .as_deref() + .is_none_or(|stable| stays(stable, dialect, None)), + format!("{name} layout drifted: {sql}"), + ), + ( + wrapped.is_some() == actual.is_some(), + format!("{name} wrapped outcome differs: {sql}"), + ), + ( + wrapped + .as_deref() + .is_none_or(|wrapped| stays(wrapped, dialect, Some(CORPUS_LINE_WIDTH))), + format!("{name} wrapping drifted: {sql}"), + ), + ]; + EntryOutcome { + formatted: actual.is_some(), + departed: departure.is_some(), + failures: checks + .into_iter() + .filter(|(passed, _)| !passed) + .map(|(_, failure)| failure) + .collect(), + } +} + +/// Format every corpus query of one dialect; return the formatted, refused and departing counts and failures. +pub(super) fn check_dialect_corpus( + corpus: &[Value], + test_case: &FormatCorpusTestCase, +) -> CorpusOutcome { + let outcomes: Vec = corpus + .iter() + .filter(|entry| entry["dialect"] == test_case.dialect) + .map(|entry| check_entry(entry, test_case)) + .collect(); + let formatted = outcomes.iter().filter(|outcome| outcome.formatted).count(); + CorpusOutcome { + formatted, + refused: outcomes.len() - formatted, + departed: outcomes.iter().filter(|outcome| outcome.departed).count(), + failures: outcomes + .into_iter() + .flat_map(|outcome| outcome.failures) + .collect(), + } +} + +/// The failure constant `reason` starts with, or all of `reason` when it starts with none of them. +pub(super) fn refusal_start(reason: &str) -> &str { + [PARSE_FAILURE, DEPTH_FAILURE, TYPE_DEPTH_FAILURE] + .into_iter() + .find(|failure| reason.starts_with(failure)) + .unwrap_or(reason) +} diff --git a/crates/sqlquack-format/src/format/tests/test_format_corpus.rs b/crates/sqlquack-format/src/format/tests/test_format_corpus.rs new file mode 100644 index 000000000..cdd5c0fa8 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/test_format_corpus.rs @@ -0,0 +1,131 @@ +use serde_json::Value; + +use crate::format::tests::helpers::check_dialect_corpus; +use crate::format::tests::test_types; + +const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); + +#[test] +fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stable() { + let test_cases = [test_types::FormatCorpusTestCase { + description: "Snowflake identity corpus", + dialect: "snowflake", + expected_formatted: 427, + expected_refused: 8, + departures: &[ + test_types::Departure { + sql: "WITH t (SELECT 1 AS c) SELECT c FROM t", + formatted: None, + reason: "Snowflake rejects a CTE body without AS, so SQLQuack's parser refuses it", + receipt: "sf.query.with.select.k8b29420d51a4", + }, + test_types::Departure { + sql: "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", + formatted: None, + reason: "Snowflake rejects a quoted date part in the FROM form of DATE_PART, so SQLQuack's parser refuses it", + receipt: "sf.functions.date-part.date-part-from.k10b598e047b5", + }, + test_types::Departure { + sql: "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", + formatted: Some( + "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data: level1 : level2 : level3::VARIANT\nFROM t", + ), + reason: "Snowflake names an unaliased select item by its text, spaces included, so the item keeps its spacing; SQLClean respaced it", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + formatted: Some( + "SELECT\n RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + formatted: Some( + "SELECT\n TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + formatted: Some("SELECT\n ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + formatted: Some( + "SELECT\n AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT {fn DAYNAME('2022-5-13')}", + formatted: Some("SELECT {fn DAYNAME('2022-5-13')}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT {fn LOG(5)}", + formatted: Some("SELECT {fn LOG(5)}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT {fn CEILING(5.3)}", + formatted: Some("SELECT {fn CEILING(5.3)}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + formatted: Some( + "SELECT\n TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + formatted: Some( + "SELECT\n SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + test_types::Departure { + sql: "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + formatted: Some( + "SELECT\n SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", + }, + ], + }]; + let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).unwrap_or_default(); + for test_case in test_cases { + let outcome = check_dialect_corpus(&corpus, &test_case); + assert_eq!( + ( + outcome.formatted, + outcome.refused, + outcome.departed, + outcome.failures.len() + ), + ( + test_case.expected_formatted, + test_case.expected_refused, + test_case.departures.len(), + 0 + ), + "{}: {}", + test_case.description, + outcome.failures.join("\n\n") + ); + } +} diff --git a/crates/sqlquack-format/src/format/tests/test_format_sql.rs b/crates/sqlquack-format/src/format/tests/test_format_sql.rs new file mode 100644 index 000000000..da3e5f0a7 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/test_format_sql.rs @@ -0,0 +1,413 @@ +use sqlquack_syntax::Dialect; + +use crate::format::constants::{ + DEPTH_FAILURE, PARSE_FAILURE, SEPARATION_FAILURE, TYPE_DEPTH_FAILURE, + UNSUPPORTED_DIALECT_FAILURE, +}; +use crate::format::main::format_sql::format_sql; +use crate::format::models::FormatOptions; +use crate::format::tests::helpers::refusal_start; +use crate::format::tests::test_types; + +#[test] +fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_refused() { + let test_cases = [ + test_types::FormatSqlTestCase { + description: "struct field names spelled like keywords keep their case in a column type", + sql: "create table t (x object(type varchar, first int, key int))", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("CREATE TABLE t(x object(type varchar, first int, key int))"), + }, + test_types::FormatSqlTestCase { + description: "an unquoted PUT file URI is not split at its slashes", + sql: "PUT file:///data/x.csv @s1/p1/", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "an unquoted GET file URI is not split at its slashes", + sql: "GET @s1/p1/ file:///tmp/", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a hex-looking number is not split into a number and an alias", + sql: "SELECT 0x1F", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a MATCH_RECOGNIZE pattern keeps its reluctant quantifier `*?`", + sql: "SELECT * FROM t MATCH_RECOGNIZE (ORDER BY b PATTERN (A{2,} B*? C+) DEFINE A AS a > 0)", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT *\nFROM t MATCH_RECOGNIZE(ORDER BY b PATTERN(A{2,} B*? C+) DEFINE A AS a > 0)", + ), + }, + test_types::FormatSqlTestCase { + description: "a comparison written without spaces is spaced", + sql: "select a from t where x=1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a\nFROM t\nWHERE x = 1"), + }, + test_types::FormatSqlTestCase { + description: "a join condition written without spaces is spaced", + sql: "select a from t join u on t.id=u.id", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a\nFROM t\nJOIN u\n ON t.id = u.id"), + }, + test_types::FormatSqlTestCase { + description: "two-character comparisons stay whole", + sql: "select a from t where x<=1 and y!=2 and z<>3", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a\nFROM t\nWHERE\n x <= 1\n AND y != 2\n AND z <> 3"), + }, + test_types::FormatSqlTestCase { + description: "an operator chain and a concatenation are spaced", + sql: "select a x from t where a||b = 'x' and c+d*e-f = 1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT a x\nFROM t\nWHERE\n a || b = 'x'\n AND c + d * e - f = 1", + ), + }, + test_types::FormatSqlTestCase { + description: "exponent signs stay inside their numbers", + sql: "select a y from t where y=1e-3 and z=1E+5", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a y\nFROM t\nWHERE\n y = 1e-3\n AND z = 1E+5"), + }, + test_types::FormatSqlTestCase { + description: "stars that are not multiplication stay touching", + sql: "select t.*, count(*) c from t where x=1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT\n t.*,\n count(*) c\nFROM t\nWHERE x = 1"), + }, + test_types::FormatSqlTestCase { + description: "path and cast accessors stay touching", + sql: "select x:y z, x::int w from t where x::int=1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT\n x:y z,\n x::int w\nFROM t\nWHERE x::int = 1"), + }, + test_types::FormatSqlTestCase { + description: "a named argument arrow is not split", + sql: "select f(a=>1) y from t", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a doubled minus keeps its space", + sql: "select a y from t where x = - -1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a y\nFROM t\nWHERE x = - -1"), + }, + test_types::FormatSqlTestCase { + description: "a multiplication beside a comment keeps the comment apart", + sql: "select a y from t where x=a/*c*/*b", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a y\nFROM t\nWHERE\n x = a /*c*/ * b"), + }, + test_types::FormatSqlTestCase { + description: "a star touching a comment opener is refused", + sql: "select a y from t where x*/*c*/b=1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "an operator beside a bind or session variable is refused", + sql: "select a y from t where x=$1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a number touching a keyword is refused", + sql: "select a y from t where x=1and y=2", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "operators in a literal-only chain without parse spans are refused", + sql: "select a y from t where 1+2+3=6", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Err(SEPARATION_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a long array breaks just inside its brackets", + sql: "select ['aaaaaaaaaa','bbbbbbbbbb','cccccccccc','dddddddddd'] y from t", + line_width: Some(40), + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT\n [\n 'aaaaaaaaaa',\n 'bbbbbbbbbb',\n 'cccccccccc',\n 'dddddddddd'\n ] y\nFROM t", + ), + }, + test_types::FormatSqlTestCase { + description: "PIVOT IN values keep their spacing, which names the pivot columns", + sql: "SELECT * FROM t PIVOT (SUM(v) FOR k IN (1+1, 2-1)) ORDER BY 1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT *\nFROM t PIVOT(SUM(v) FOR k IN (1+1, 2-1))\nORDER BY\n 1", + ), + }, + test_types::FormatSqlTestCase { + description: "PIVOT IN concatenations keep their spacing", + sql: "SELECT * FROM t PIVOT (SUM(v) FOR k IN ('a'||'b', 'c')) ORDER BY 1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT *\nFROM t PIVOT(SUM(v) FOR k IN ('a'||'b', 'c'))\nORDER BY\n 1", + ), + }, + test_types::FormatSqlTestCase { + description: "PIVOT IN values keep a spaced unary minus", + sql: "SELECT * FROM t PIVOT (SUM(v) FOR k IN (- 1, 2)) ORDER BY 1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT *\nFROM t PIVOT(SUM(v) FOR k IN (- 1, 2))\nORDER BY\n 1"), + }, + test_types::FormatSqlTestCase { + description: "select items without an alias keep their text, which names their column", + sql: "SELECT x FROM (SELECT a+b, - 1, t . c AS d FROM t) WHERE \"A+B\" > 1", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "SELECT x\nFROM (\n SELECT\n a+b,\n - 1,\n t.c AS d\n FROM t\n)\nWHERE \"A+B\" > 1", + ), + }, + test_types::FormatSqlTestCase { + description: "a CTAS select item without an alias keeps its keyword case", + sql: "CREATE OR REPLACE TABLE t2 AS SELECT count( distinct g ), g FROM t", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok( + "CREATE OR REPLACE TABLE t2 AS SELECT\n count( distinct g ),\n g\nFROM t", + ), + }, + test_types::FormatSqlTestCase { + description: "a second trailing line comment keeps its indentation", + sql: "SELECT a FROM t -- one\n-- two", + line_width: None, + dialect: Dialect::Snowflake, + expected_outcome: Ok("SELECT a\nFROM t -- one\n-- two"), + }, + test_types::FormatSqlTestCase { + description: "DuckDB is refused until it has a format gate", + sql: "SELECT a FROM t", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Err(UNSUPPORTED_DIALECT_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "PostgreSQL is refused until it has a format gate", + sql: "SELECT a FROM t", + line_width: None, + dialect: Dialect::PostgreSQL, + expected_outcome: Err(UNSUPPORTED_DIALECT_FAILURE), + }, + ]; + for test_case in test_cases { + let options = FormatOptions { + line_width: test_case.line_width, + ..FormatOptions::new(test_case.dialect) + }; + let actual = format_sql(test_case.sql, &options); + assert_eq!( + actual + .map(|formatted| formatted.sql) + .map_err(|refusal| refusal.reason), + test_case + .expected_outcome + .map(str::to_string) + .map_err(str::to_string), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_without_exhausting_the_stack() + { + let test_cases = [ + test_types::DeepSqlTestCase { + description: "1,500 parentheses around a FROM source are refused by the parser's nesting guard", + sql: || format!("SELECT * FROM {}t{}", "(".repeat(1_500), ")".repeat(1_500)), + expected_refusal: Some(PARSE_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "1,500 nested subqueries are refused by the parser's depth guard", + sql: || { + format!( + "{}SELECT 1{}", + "SELECT * FROM (".repeat(1_500), + ") AS q".repeat(1_500) + ) + }, + expected_refusal: Some(PARSE_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "1,500 nested NOTs are refused by the parser's depth guard", + sql: || format!("SELECT {}TRUE", "NOT ".repeat(1_500)), + expected_refusal: Some(PARSE_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a chain of 5,000 added terms is refused before its tree is walked", + sql: || format!("SELECT {}", vec!["1"; 5_000].join(" + ")), + expected_refusal: Some(DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a chain of 513 added terms, one level past the limit, is refused", + sql: || format!("SELECT {}", vec!["1"; 513].join(" + ")), + expected_refusal: Some(DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to a type with 300 `[]` suffixes is refused before parsing", + sql: || format!("SELECT CAST(NULL AS INT{})", "[]".repeat(300)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to a type with 300 `[3]` suffixes is refused before parsing", + sql: || format!("SELECT CAST(NULL AS INT{})", "[3]".repeat(300)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a column of a type with 65 `[]` suffixes, one past the limit, is refused", + sql: || format!("CREATE TABLE t (c INT{})", "[]".repeat(65)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a column of a type with 64 `[]` suffixes, at the limit, is laid out", + sql: || format!("CREATE TABLE t (c INT{})", "[]".repeat(64)), + expected_refusal: None, + }, + test_types::DeepSqlTestCase { + description: "a type alternating angle brackets with suffix runs, depth 182, is refused", + sql: || { + format!( + "SELECT CAST(NULL AS ARRAY", + "[]".repeat(60) + ">" + &"[]".repeat(60) + ) + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to 2 angle brackets around 63 suffixes, depth 65, is refused", + sql: || format!("SELECT CAST(NULL AS ARRAY>)", "[]".repeat(63)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to 2 angle brackets around 62 suffixes, depth 64, is laid out", + sql: || format!("SELECT CAST(NULL AS ARRAY>)", "[]".repeat(62)), + expected_refusal: None, + }, + test_types::DeepSqlTestCase { + description: "a struct of 100 one-suffix fields stays shallow and is laid out", + sql: || { + format!( + "SELECT CAST(NULL AS STRUCT<{}>)", + (0..100) + .map(|field| format!("f{field} INT[]")) + .collect::>() + .join(", ") + ) + }, + expected_refusal: None, + }, + test_types::DeepSqlTestCase { + description: "a type alternating parenthesised constructors with suffix runs, depth 182, is refused", + sql: || { + format!( + "SELECT CAST(NULL AS ARRAY(ARRAY(INT{}{})", + "[]".repeat(60) + ")", + "[]".repeat(60) + ")" + &"[]".repeat(60) + ) + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a type alternating quoted parenthesised constructors with suffix runs, depth 182, is refused", + sql: || { + format!( + "SELECT CAST(NULL AS \"ARRAY\"(\"ARRAY\"(INT{}{})", + "[]".repeat(60) + ")", + "[]".repeat(60) + ")" + &"[]".repeat(60) + ) + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a column alternating quoted STRUCT<...> and ARRAY<...> with suffix runs is refused", + sql: || { + let run = "[]".repeat(40); + format!("CREATE TABLE t (c \"STRUCT\"{run}>{run})") + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a column alternating STRUCT(...), MAP(...) and ARRAY<...> with suffix runs is refused", + sql: || { + let run = "[]".repeat(30); + format!( + "CREATE TABLE t (c STRUCT(a MAP(INT, ARRAY{run}){run}){run})" + ) + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to 2 parenthesised constructors around 63 suffixes, depth 65, is refused", + sql: || format!("SELECT CAST(NULL AS ARRAY(ARRAY(INT{})))", "[]".repeat(63)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a cast to 2 parenthesised constructors around 62 suffixes, depth 64, is laid out", + sql: || format!("SELECT CAST(NULL AS ARRAY(ARRAY(INT{})))", "[]".repeat(62)), + expected_refusal: None, + }, + test_types::DeepSqlTestCase { + description: "a cast to 300 nested ARRAY<...> types is refused before parsing", + sql: || { + format!( + "SELECT CAST(NULL AS {}INT{})", + "ARRAY<".repeat(300), + ">".repeat(300) + ) + }, + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, + test_types::DeepSqlTestCase { + description: "a chain of 512 added terms, at the limit, is laid out", + sql: || format!("SELECT {}", vec!["1"; 512].join(" + ")), + expected_refusal: None, + }, + ]; + for test_case in test_cases { + let outcome = format_sql(&(test_case.sql)(), &FormatOptions::new(Dialect::Snowflake)); + assert_eq!( + outcome + .err() + .map(|refusal| refusal_start(&refusal.reason).to_string()), + test_case.expected_refusal.map(str::to_string), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-format/src/format/tests/test_types.rs b/crates/sqlquack-format/src/format/tests/test_types.rs new file mode 100644 index 000000000..18eb4d6c0 --- /dev/null +++ b/crates/sqlquack-format/src/format/tests/test_types.rs @@ -0,0 +1,55 @@ +//! Test case types of format. + +/// One dialect of the format corpus with its expected formatted and refused counts. +pub(super) struct FormatCorpusTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static str, + pub(super) expected_formatted: usize, + pub(super) expected_refused: usize, + /// Corpus queries whose SQLQuack outcome deliberately differs from SQLClean's. + pub(super) departures: &'static [Departure], +} + +/// A corpus query whose SQLQuack outcome deliberately differs from SQLClean's recorded one. +pub(super) struct Departure { + pub(super) sql: &'static str, + /// SQLQuack's layout; `None` when SQLQuack refuses the query. + pub(super) formatted: Option<&'static str>, + pub(super) reason: &'static str, + /// The recorded Snowflake evidence for SQLQuack's outcome: a `coverage/syntax` receipt or a `cases` case. + pub(super) receipt: &'static str, +} + +/// The formatted, refused and departing counts of one corpus dialect, and every check that failed. +pub(super) struct CorpusOutcome { + pub(super) formatted: usize, + pub(super) refused: usize, + /// Corpus queries that matched a departure. + pub(super) departed: usize, + pub(super) failures: Vec, +} + +/// One corpus query's outcome: whether it was formatted or departs, and the checks it failed. +pub(super) struct EntryOutcome { + pub(super) formatted: bool, + pub(super) departed: bool, + pub(super) failures: Vec, +} + +/// A generated, deeply nested Snowflake query, and the refusal it gets, if any. +pub(super) struct DeepSqlTestCase { + pub(super) description: &'static str, + pub(super) sql: fn() -> String, + /// The start of the refusal reason; `None` when the query is laid out. + pub(super) expected_refusal: Option<&'static str>, +} + +/// One query formatted through `format_sql`, with its laid-out SQL or refusal reason. +pub(super) struct FormatSqlTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) dialect: sqlquack_syntax::Dialect, + pub(super) line_width: Option, + /// The laid-out SQL, or the reason it is refused. + pub(super) expected_outcome: Result<&'static str, &'static str>, +} diff --git a/crates/sqlquack-format/src/lib.rs b/crates/sqlquack-format/src/lib.rs new file mode 100644 index 000000000..adc5c3a63 --- /dev/null +++ b/crates/sqlquack-format/src/lib.rs @@ -0,0 +1,8 @@ +//! SQL layout that changes only token spacing and keyword case, ported from SQLClean's formatter. + +#![forbid(unsafe_code)] + +pub mod format; + +pub use crate::format::main::format_sql::format_sql; +pub use crate::format::models::{FormatOptions, FormatRefusal, Formatted}; diff --git a/docs/architecture.md b/docs/architecture.md index 31df1840d..d12093816 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,6 +36,7 @@ a parameter of every entry point; Snowflake is the only one implemented. | `sqlquack-rules` | The conversion rules, their type conditions, and declared refusals. | | `sqlquack-duckdb` | DuckDB building blocks shared by rules, and DuckDB's own parse-and-bind check of every converted statement. | | `sqlquack-policy` | The policy file `sqlquack.toml`: schema, validation and type-pattern matching. | +| `sqlquack-format` | The SQL formatter: lays SQL out over lines, changing only the whitespace between tokens and keyword case. A layout guide prints the authored tokens as a parse-tree pretty printer would; the output is refused unless it keeps every token, keeps comments attached and is idempotent. It builds on `sqlquack-syntax` alone. | | `sqlquack-compare` | The strict comparator between a recorded result and a DuckDB replay or a fresh recording. | | `sqlquack-evidence` | Cases, case hashes, pending files, stored results, DuckDB replay, verification and audits. | | `sqlquack` | The `sqlquack` command line and the library entry points: the pipeline, the census, the index and the evidence commands. | From 5c80d8d677a6ce6e879009d5977b4793949c9210 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:46:54 +0000 Subject: [PATCH 19/22] feat(format): lay out DuckDB with sqlquack-format Adds DuckDB to the formatter, gated on SQLGlot's DuckDB formatting fixtures, and sqlquack-format-check, which compares the result columns DuckDB reports for the original and formatted SQL. --- Cargo.lock | 10 + Cargo.toml | 1 + crates/sqlquack-format-check/Cargo.toml | 21 ++ .../src/columns/_helpers/describe.rs | 76 ++++ .../src/columns/_helpers/mod.rs | 3 + .../src/columns/constants.rs | 4 + .../src/columns/main/compare_columns.rs | 35 ++ .../src/columns/main/mod.rs | 3 + .../sqlquack-format-check/src/columns/mod.rs | 9 + .../src/columns/models.rs | 14 + .../src/columns/tests.rs | 4 + .../src/columns/tests/test_compare_columns.rs | 104 ++++++ .../src/columns/tests/test_types.rs | 13 + crates/sqlquack-format-check/src/lib.rs | 8 + .../src/format/_helpers/guide.rs | 135 ++++++- .../src/format/_helpers/pipeline.rs | 12 +- .../src/format/_helpers/touching.rs | 17 +- .../sqlquack-format/src/format/constants.rs | 27 +- .../src/format/tests/test_format_corpus.rs | 336 ++++++++++++------ .../src/format/tests/test_format_sql.rs | 92 ++++- .../src/format/tests/test_types.rs | 6 +- docs/architecture.md | 1 + fensu.toml | 1 + 23 files changed, 790 insertions(+), 142 deletions(-) create mode 100644 crates/sqlquack-format-check/Cargo.toml create mode 100644 crates/sqlquack-format-check/src/columns/_helpers/describe.rs create mode 100644 crates/sqlquack-format-check/src/columns/_helpers/mod.rs create mode 100644 crates/sqlquack-format-check/src/columns/constants.rs create mode 100644 crates/sqlquack-format-check/src/columns/main/compare_columns.rs create mode 100644 crates/sqlquack-format-check/src/columns/main/mod.rs create mode 100644 crates/sqlquack-format-check/src/columns/mod.rs create mode 100644 crates/sqlquack-format-check/src/columns/models.rs create mode 100644 crates/sqlquack-format-check/src/columns/tests.rs create mode 100644 crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs create mode 100644 crates/sqlquack-format-check/src/columns/tests/test_types.rs create mode 100644 crates/sqlquack-format-check/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 542d6a3eb..80fd1c64c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1416,6 +1416,16 @@ dependencies = [ "sqlquack-syntax", ] +[[package]] +name = "sqlquack-format-check" +version = "0.2.0" +dependencies = [ + "duckdb", + "serde_json", + "sqlquack-format", + "sqlquack-syntax", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 30eb6684c..c637ce9db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/sqlquack-syntax-coverage", "crates/sqlquack-syntax-postgres", "crates/sqlquack-format", + "crates/sqlquack-format-check", "crates/sqlquack-syntax-bigquery", "crates/sqlquack-python", ] diff --git a/crates/sqlquack-format-check/Cargo.toml b/crates/sqlquack-format-check/Cargo.toml new file mode 100644 index 000000000..3b6d3fdc1 --- /dev/null +++ b/crates/sqlquack-format-check/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sqlquack-format-check" +version.workspace = true +description = "Development check that formatting leaves DuckDB's output columns unchanged; never a runtime dependency." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish = false +rust-version.workspace = true + +[dependencies] +duckdb.workspace = true +sqlquack-format.workspace = true +sqlquack-syntax.workspace = true + +[dev-dependencies] +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-format-check/src/columns/_helpers/describe.rs b/crates/sqlquack-format-check/src/columns/_helpers/describe.rs new file mode 100644 index 000000000..6fcc8d529 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/_helpers/describe.rs @@ -0,0 +1,76 @@ +//! DuckDB's description of a query's output columns, before and after the formatter lays it out. + +use sqlquack_format::{FormatOptions, format_sql}; +use sqlquack_syntax::Dialect; + +/// One query at one line width: refused, or laid out with its source bound or not, and any change in its columns. +pub(crate) enum LayoutCheck { + Refused, + Bound(Option), + Unbound(Option), +} + +/// `sql` laid out for DuckDB, or the formatter's reason for refusing it. +fn duckdb_layout(sql: &str, line_width: Option) -> Result { + let options = FormatOptions { + line_width, + ..FormatOptions::new(Dialect::DuckDB) + }; + format_sql(sql, &options) + .map(|formatted| formatted.sql) + .map_err(|refusal| refusal.reason) +} + +/// The output columns DuckDB binds `sql` to, as names and types, or DuckDB's reason it does not bind. +fn described(connection: &duckdb::Connection, sql: &str) -> Result, String> { + let mut statement = connection + .prepare(&format!( + "DESCRIBE {}", + sql.trim_end().trim_end_matches(';') + )) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + }) + .map_err(|error| error.to_string())?; + rows.collect::, _>>() + .map_err(|error| error.to_string()) +} + +/// Whether `after` binds to other columns than `before`, or binds where it does not; reasons for not binding name moved line positions, so they are not compared. +fn binds_differently( + before: &Result, String>, + after: &Result, String>, +) -> bool { + match (before, after) { + (Ok(before), Ok(after)) => before != after, + (Err(_), Err(_)) => false, + _ => true, + } +} + +/// Lay out `sql` at `line_width` and compare the columns DuckDB binds the layout to with the source's. +pub(crate) fn layout_check( + connection: &duckdb::Connection, + sql: &str, + line_width: Option, +) -> LayoutCheck { + let layout = duckdb_layout(sql, line_width); + let before = described(connection, sql); + match layout { + Err(_) => LayoutCheck::Refused, + Ok(layout) => { + let after = described(connection, &layout); + let change = binds_differently(&before, &after).then(|| { + format!( + "{sql}\n--- laid out at {line_width:?}\n{layout}\n--- before {before:?}\n--- after {after:?}" + ) + }); + match before { + Ok(_) => LayoutCheck::Bound(change), + Err(_) => LayoutCheck::Unbound(change), + } + } + } +} diff --git a/crates/sqlquack-format-check/src/columns/_helpers/mod.rs b/crates/sqlquack-format-check/src/columns/_helpers/mod.rs new file mode 100644 index 000000000..54388cee4 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Helpers of columns. + +pub(crate) mod describe; diff --git a/crates/sqlquack-format-check/src/columns/constants.rs b/crates/sqlquack-format-check/src/columns/constants.rs new file mode 100644 index 000000000..1f9d97d40 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/constants.rs @@ -0,0 +1,4 @@ +//! Constants of columns. + +/// The line width the format corpus is also wrapped at, besides no width. +pub const CORPUS_LINE_WIDTH: usize = 40; diff --git a/crates/sqlquack-format-check/src/columns/main/compare_columns.rs b/crates/sqlquack-format-check/src/columns/main/compare_columns.rs new file mode 100644 index 000000000..5fbef7026 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/main/compare_columns.rs @@ -0,0 +1,35 @@ +//! `compare_columns`: bind each query and its DuckDB layouts in DuckDB and compare their output columns. + +use crate::columns::_helpers::describe::{LayoutCheck, layout_check}; +use crate::columns::constants::CORPUS_LINE_WIDTH; +use crate::columns::models::ColumnsOutcome; + +/// Lay out every query for DuckDB at no width and the corpus width; report layouts whose output columns differ from the source's. +pub fn compare_columns(connection: &duckdb::Connection, queries: &[&str]) -> ColumnsOutcome { + let checks: Vec = queries + .iter() + .flat_map(|sql| [(*sql, None), (*sql, Some(CORPUS_LINE_WIDTH))]) + .map(|(sql, line_width)| layout_check(connection, sql, line_width)) + .collect(); + ColumnsOutcome { + bound: checks + .iter() + .filter(|check| matches!(check, LayoutCheck::Bound(_))) + .count(), + unbound: checks + .iter() + .filter(|check| matches!(check, LayoutCheck::Unbound(_))) + .count(), + refused: checks + .iter() + .filter(|check| matches!(check, LayoutCheck::Refused)) + .count(), + changes: checks + .into_iter() + .filter_map(|check| match check { + LayoutCheck::Bound(change) | LayoutCheck::Unbound(change) => change, + LayoutCheck::Refused => None, + }) + .collect(), + } +} diff --git a/crates/sqlquack-format-check/src/columns/main/mod.rs b/crates/sqlquack-format-check/src/columns/main/mod.rs new file mode 100644 index 000000000..e87f25256 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/main/mod.rs @@ -0,0 +1,3 @@ +//! Entry points of columns. + +pub mod compare_columns; diff --git a/crates/sqlquack-format-check/src/columns/mod.rs b/crates/sqlquack-format-check/src/columns/mod.rs new file mode 100644 index 000000000..ecfa8ecc1 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/mod.rs @@ -0,0 +1,9 @@ +//! The output columns DuckDB binds a query to, before and after the formatter lays it out. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-format-check/src/columns/models.rs b/crates/sqlquack-format-check/src/columns/models.rs new file mode 100644 index 000000000..9dd5788f9 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/models.rs @@ -0,0 +1,14 @@ +//! Data models of columns. + +/// Layouts counted by whether DuckDB binds their source, and each one whose columns changed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ColumnsOutcome { + /// Layouts whose source DuckDB binds, so their columns were compared. + pub bound: usize, + /// Layouts whose source DuckDB does not bind, such as ones naming missing tables. + pub unbound: usize, + /// Layouts the formatter refused. + pub refused: usize, + /// Each layout whose bound columns, or whether it binds, differ from its source. + pub changes: Vec, +} diff --git a/crates/sqlquack-format-check/src/columns/tests.rs b/crates/sqlquack-format-check/src/columns/tests.rs new file mode 100644 index 000000000..a7adaea3d --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_compare_columns.rs"] +mod test_compare_columns; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs b/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs new file mode 100644 index 000000000..699945ccf --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs @@ -0,0 +1,104 @@ +use std::path::Path; + +use serde_json::Value; + +use crate::columns::main::compare_columns::compare_columns; +use crate::columns::tests::test_types; + +/// Lets DuckDB 2.0 bind `->` lambdas, which it otherwise refuses at bind. +const SINGLE_ARROW_LAMBDAS: &str = "SET lambda_syntax = 'ENABLE_SINGLE_ARROW'"; +/// The format corpus, beside the formatter's own corpus test. +const FORMAT_CORPUS: &str = "../sqlquack-format/src/format/tests/fixtures/format_corpus.json"; + +#[test] +fn given_duckdb_queries_when_laid_out_then_duckdb_binds_the_same_output_columns() { + let test_cases = [ + test_types::CompareColumnsTestCase { + description: "DuckDB corpus subset", + corpus_dialect: "duckdb", + queries: &[], + expected_bound: 188, + expected_unbound: 190, + expected_refused: 0, + }, + test_types::CompareColumnsTestCase { + description: "operator, spacing and keyword-case inputs over inline relations", + corpus_dialect: "", + queries: &[ + "select a+b, a-b, a*b, a=b, a<>b, a!=b, a<=b, a>=b, ab from (select 1 as a, 2 as b)", + "select s||t, upper( s ), lower(s), - 1, -1, - -1, 1e-3, 1E+5, .5e-2 from (select 'x' as s, 'y' as t)", + "select count(*), Count( distinct a ), max( a ), sum(a)+1 from (select 1 as a)", + "select case when a=1 then 'x' else 'y' end, a between 1 and 2, a in (1,2), not a=1, a is null from (select 1 as a)", + "select cast(a as varchar), a::int, cast( a AS bigint ) from (select 1 as a)", + "select [1,2][1], {'k':1}, [ 1 ], [1,2,3][1:2] from (select 1)", + "select a+b\n + 3 from (select 1 as a, 2 as b)", + "select \"a+b\" from (select a+b from (select 1 as a, 2 as b))", + "select \"(a + b)\" from (select a+b from (select 1 as a, 2 as b))", + "select distinct A from (select 1 as a) where a=1 order by a", + "select 1 x, 2 AS y, coalesce( a , 0 ) from (select null as a)", + "select list_transform([1,2], x -> x+1), list_filter([1,2], lambda x: x>1)", + "select interval 1 day, date '2024-01-01' + interval 1 day, current_date", + "select t.a, t . a, t.* from (select 1 as a) as t", + "select a+b as c, a+b c from (select 1 as a, 2 as b)", + "with q as (select a*2 from (select 1 as a)) select * from q", + "select * from (select a+b from (select 1 as a, 2 as b)) union all select 3", + "select (a), ((a+b)), (select 1) from (select 1 as a, 2 as b)", + "select a % b, a // b, a / b, 2 ** 3, a & b, a | b, a << 1 from (select 7 as a, 2 as b)", + "select upper('a') || lower('B'), 'a' like 'a%', 'a' ilike 'A', 'a' similar to 'a' from (select 1)", + "select sum(a) over (order by a), row_number() over () from (select 1 as a)", + "select a::varchar || 'x', a::int+1 from (select 1 as a)", + "select * from (select a+b, a*b from (select 1 as a, 2 as b))", + "select - @a from (select -2 as a)", + "select + @a from (select -2 as a)", + "select @ +1", + "select @ -1", + "select @ - a from (select 2 as a)", + "select a - @b, @a, @(-1) from (select 1 as a, -2 as b)", + ], + expected_bound: 42, + expected_unbound: 4, + expected_refused: 12, + }, + ]; + let corpus_text = + std::fs::read_to_string(Path::new(env!("CARGO_MANIFEST_DIR")).join(FORMAT_CORPUS)) + .unwrap_or_default(); + let corpus: Vec = serde_json::from_str(&corpus_text).unwrap_or_default(); + let connection = duckdb::Connection::open_in_memory().and_then(|connection| { + connection + .execute_batch(SINGLE_ARROW_LAMBDAS) + .map(|()| connection) + }); + for test_case in test_cases { + let queries: Vec<&str> = corpus + .iter() + .filter(|entry| entry["dialect"] == test_case.corpus_dialect) + .filter_map(|entry| entry["sql"].as_str()) + .chain(test_case.queries.iter().copied()) + .collect(); + let outcome = connection + .as_ref() + .map(|connection| compare_columns(connection, &queries)) + .map_err(ToString::to_string); + assert_eq!( + outcome.as_ref().map(|outcome| ( + outcome.bound, + outcome.unbound, + outcome.refused, + outcome.changes.len() + )), + Ok(( + test_case.expected_bound, + test_case.expected_unbound, + test_case.expected_refused, + 0 + )), + "{}: {}", + test_case.description, + outcome + .as_ref() + .map(|outcome| outcome.changes.join("\n\n")) + .unwrap_or_default() + ); + } +} diff --git a/crates/sqlquack-format-check/src/columns/tests/test_types.rs b/crates/sqlquack-format-check/src/columns/tests/test_types.rs new file mode 100644 index 000000000..fe173f294 --- /dev/null +++ b/crates/sqlquack-format-check/src/columns/tests/test_types.rs @@ -0,0 +1,13 @@ +//! Test case types of columns. + +/// Queries laid out for DuckDB whose output columns DuckDB must bind the same before and after. +pub(super) struct CompareColumnsTestCase { + pub(super) description: &'static str, + /// The format corpus dialect whose queries are included; empty for none. + pub(super) corpus_dialect: &'static str, + pub(super) queries: &'static [&'static str], + /// Layouts, at no line width and at the corpus width, whose source DuckDB binds. + pub(super) expected_bound: usize, + pub(super) expected_unbound: usize, + pub(super) expected_refused: usize, +} diff --git a/crates/sqlquack-format-check/src/lib.rs b/crates/sqlquack-format-check/src/lib.rs new file mode 100644 index 000000000..4815a2817 --- /dev/null +++ b/crates/sqlquack-format-check/src/lib.rs @@ -0,0 +1,8 @@ +//! Development check that formatting leaves DuckDB's output columns unchanged; never a runtime dependency. + +#![forbid(unsafe_code)] + +pub mod columns; + +pub use crate::columns::main::compare_columns::compare_columns; +pub use crate::columns::models::ColumnsOutcome; diff --git a/crates/sqlquack-format/src/format/_helpers/guide.rs b/crates/sqlquack-format/src/format/_helpers/guide.rs index 0a9499f83..b8a612477 100644 --- a/crates/sqlquack-format/src/format/_helpers/guide.rs +++ b/crates/sqlquack-format/src/format/_helpers/guide.rs @@ -10,11 +10,12 @@ use crate::format::_helpers::lexemes::{ Lexeme, bracket_partners, ends_operand, is_keyword, is_unquoted_word, }; use crate::format::constants::{ - ALIAS_FOLLOWERS, CASE_BRANCHES, CLAUSE_KEYWORDS, DUCKDB_INLINE_CALLS, GROUPING_CALLS, - GUIDE_INDENT, GUIDE_WIDTH, JOIN_PREFIXES, PLACEHOLDER_WIDTH, POSTFIX_CAST_WIDTH, - POSTGRES_INLINE_CALLS, SAMPLE_ROW_KEYWORD, SEMANTIC_VIEW_CALL, SEMANTIC_VIEW_CLAUSES, - SET_OPERATOR_MODIFIERS, SET_OPERATORS, SNOWFLAKE_INLINE_CALLS, SPACED_BEFORE_PARENTHESIS, - STRUCT_TYPE_WORDS, TYPE_CONTINUATIONS, TYPED_LITERALS, + ALIAS_FOLLOWERS, CASE_BRANCHES, CLAUSE_KEYWORDS, COLUMNS_CALL, DUCKDB_INLINE_CALLS, + EXPRESSION_LEADERS, GROUPING_CALLS, GUIDE_INDENT, GUIDE_WIDTH, JOIN_PREFIXES, LAMBDA_KEYWORD, + PLACEHOLDER_WIDTH, POSTFIX_CAST_WIDTH, POSTGRES_INLINE_CALLS, SAMPLE_KEYWORDS, + SAMPLE_ROW_KEYWORD, SEMANTIC_VIEW_CALL, SEMANTIC_VIEW_CLAUSES, SET_OPERATOR_MODIFIERS, + SET_OPERATORS, SNOWFLAKE_INLINE_CALLS, SPACED_BEFORE_PARENTHESIS, STRUCT_TYPE_WORDS, + TYPE_CONTINUATIONS, TYPED_LITERALS, }; /// The guide's text and its tokens, with byte ranges in that text. @@ -222,7 +223,15 @@ impl<'a> Guide<'a> { return ""; } if previous.kind == TokenType::Colon { - return if self.path_colon(index - 1) { "" } else { " " }; + return if self.tight_colon(index - 1) { "" } else { " " }; + } + if (previous.kind == TokenType::DAt && !self.in_stage[index - 1]) + || (previous.kind == TokenType::Star && self.is(index, COLUMNS_CALL)) + { + return ""; + } + if next.kind == TokenType::Percent && self.postfix_percent(index) { + return ""; } if matches!(previous.kind, TokenType::Dash | TokenType::Plus) && index @@ -270,6 +279,56 @@ impl<'a> Guide<'a> { || (upper == SAMPLE_ROW_KEYWORD && after_sample) } + /// Whether a `:` at `index` binds tightly: a Snowflake path, a slice, or a DuckDB prefix alias. + fn tight_colon(&self, index: usize) -> bool { + let enclosing = self.enclosing[index].map(|opener| self.lexemes[opener].kind); + self.path_colon(index) + || self.in_subscript(index) + || (self.dialect == Dialect::DuckDB + && enclosing != Some(TokenType::LBrace) + && !self.lambda_colon(index)) + } + + /// Whether token `index` is inside a subscript such as `x[1:2]`. + fn in_subscript(&self, index: usize) -> bool { + self.enclosing[index].is_some_and(|opener| { + self.lexemes[opener].kind == TokenType::LBracket + && opener > 0 + && self.lexemes[opener - 1].end == self.lexemes[opener].start + && self.operand_ends(opener - 1) + }) + } + + /// Whether the `:` at `index` ends the parameters of a DuckDB `LAMBDA x, y :`. + fn lambda_colon(&self, index: usize) -> bool { + let parameters = (0..index).rev().find(|&at| { + !matches!( + self.lexemes[at].kind, + TokenType::Var | TokenType::Identifier | TokenType::Comma + ) || self.is(at, LAMBDA_KEYWORD) + }); + parameters.is_some_and(|at| self.is(at, LAMBDA_KEYWORD)) + } + + /// Whether the `%` at `index` is DuckDB's percentage suffix, as in `LIMIT 10%`, not modulo. + fn postfix_percent(&self, index: usize) -> bool { + let after_number = self.kind(index - 1) == Some(TokenType::Number); + let sampled = index > 1 && self.is_any(index - 2, &SAMPLE_KEYWORDS); + let operand_follows = match self.kind(index + 1) { + Some(TokenType::LParen) => !sampled, + Some(kind) => matches!( + kind, + TokenType::Number + | TokenType::Var + | TokenType::Identifier + | TokenType::QuotedIdentifier + | TokenType::Dash + ), + None => false, + }; + self.dialect == Dialect::DuckDB && after_number && !operand_follows + } + /// Whether the `:` at `index` is a Snowflake path accessor such as `v:field`. fn path_colon(&self, index: usize) -> bool { self.dialect == Dialect::Snowflake @@ -415,7 +474,12 @@ impl<'a> Guide<'a> { } fn starts_query(&self, index: usize) -> bool { - self.is(index, "SELECT") || self.is(index, "WITH") + self.is(index, "SELECT") || self.is(index, "WITH") || self.starts_from_first(index) + } + + /// Whether a DuckDB FROM-first query, such as `FROM t WHERE x`, starts at `index`. + fn starts_from_first(&self, index: usize) -> bool { + self.dialect == Dialect::DuckDB && self.is(index, "FROM") } fn statements(&mut self) { @@ -491,6 +555,8 @@ impl<'a> Guide<'a> { } if self.is(first, "SELECT") { self.select(first, last, indent); + } else if self.starts_from_first(first) { + self.lay_out_from_first(first, last, indent); } else if self.is(first, "WITH") { self.query(first, last, indent); } else { @@ -516,7 +582,12 @@ impl<'a> Guide<'a> { let mut index = item + 1; if self.kind(index) == Some(TokenType::LParen) && let Some(closer) = self.partners[index] - && self.is(closer + 1, "AS") + { + index = closer + 1; + } + if self.is(index, "USING") + && self.is(index + 1, "KEY") + && let Some(closer) = self.partners.get(index + 2).copied().flatten() { index = closer + 1; } @@ -559,6 +630,7 @@ impl<'a> Guide<'a> { Some(TokenType::Comma | TokenType::RParen) ) || self.is_any(index + 1, &ALIAS_FOLLOWERS); (!aliased && self.is_any(index, &CLAUSE_KEYWORDS)) + || (self.is(index, "USING") && self.is(index + 1, "SAMPLE")) || ((self.is(index, "GROUP") || self.is(index, "ORDER")) && self.is(index + 1, "BY")) } @@ -597,7 +669,10 @@ impl<'a> Guide<'a> { self.selects += 1; for (first, end) in self.split_commas(item, projection_end) { self.gaps[first] = newline(indent + 1); - if !self.aliases[end] && !self.plain_column(first, end) { + if self.dialect == Dialect::Snowflake + && !self.aliases[end] + && !self.plain_column(first, end) + { self.verbatim.push((first, end)); } self.expr(first, end, indent + 1); @@ -616,6 +691,27 @@ impl<'a> Guide<'a> { }) } + /// Lay out a FROM-first query: its clauses, then any SELECT with the clauses after it. + fn lay_out_from_first(&mut self, from: usize, last: usize, indent: usize) { + let tops = self.tops(from + 1, last); + let select = tops.iter().copied().find(|&index| self.is(index, "SELECT")); + let end = select.map_or(last, |select| select - 1); + let clauses: Vec = std::iter::once(from) + .chain( + tops.into_iter() + .filter(|&index| index <= end && self.starts_clause(index)), + ) + .collect(); + for (position, &clause) in clauses.iter().enumerate() { + let clause_end = clauses.get(position + 1).map_or(end, |&next| next - 1); + self.clause(clause, clause_end, indent); + } + if let Some(select) = select { + self.gaps[select] = newline(indent); + self.select(select, last, indent); + } + } + fn clause(&mut self, clause: usize, end: usize, indent: usize) { self.gaps[clause] = newline(indent); match self.upper(clause).as_str() { @@ -634,6 +730,12 @@ impl<'a> Guide<'a> { self.items(body, end, indent); } "WINDOW" => self.items(clause + 1, end, indent), + "USING" => { + for index in clause + 2..=end { + self.protected[index] = true; + } + self.expr(clause + 2, end, indent); + } _ => self.expr(clause + 1, end, indent), } } @@ -807,6 +909,10 @@ impl<'a> Guide<'a> { } return; } + if self.is(opener + 1, "VALUES") { + self.expr(opener + 1, closer - 1, indent); + return; + } let Some(previous) = previous else { self.tuple(opener, closer, indent); return; @@ -822,16 +928,23 @@ impl<'a> Guide<'a> { || (self.base[opener].is_empty() && self.calls(previous) && !self.in_stage[previous]) { self.call(previous, opener, closer, indent); - } else if self.is_word(previous) || self.in_stage[previous] { + } else if (self.is_word(previous) && !self.is_any(previous, &EXPRESSION_LEADERS)) + || self.in_stage[previous] + { self.expr(opener + 1, closer - 1, indent); } else { self.tuple(opener, closer, indent); } } + /// Lay out a tuple, split past the guide width, or a parenthesised expression, always split. fn tuple(&mut self, opener: usize, closer: usize, indent: usize) { let items = self.split_commas(opener + 1, closer - 1); - let split = items.len() > 1 && self.items_width(&items) > GUIDE_WIDTH; + let split = match items.len() { + 0 => false, + 1 => true, + _ => self.items_width(&items) > GUIDE_WIDTH, + }; self.list(closer, &items, split, indent); } diff --git a/crates/sqlquack-format/src/format/_helpers/pipeline.rs b/crates/sqlquack-format/src/format/_helpers/pipeline.rs index 48868093b..659a4a959 100644 --- a/crates/sqlquack-format/src/format/_helpers/pipeline.rs +++ b/crates/sqlquack-format/src/format/_helpers/pipeline.rs @@ -12,16 +12,16 @@ use crate::format::_helpers::token_layout::{ }; use crate::format::_helpers::touching::verify_touching_tokens; use crate::format::constants::{ - DEPTH_FAILURE, MAX_TREE_DEPTH, MAX_TYPE_NESTING, NOT_IDEMPOTENT_FAILURE, PARSE_FAILURE, - SELECT_LIST_FAILURE, TOKEN_PRESERVATION_FAILURE, TYPE_CONSTRUCTOR_WORDS, TYPE_DEPTH_FAILURE, - UNSUPPORTED_DIALECT_FAILURE, UNSUPPORTED_SYNTAX_FAILURE, + DEPTH_FAILURE, FORMAT_GATED_DIALECTS, MAX_TREE_DEPTH, MAX_TYPE_NESTING, NOT_IDEMPOTENT_FAILURE, + PARSE_FAILURE, SELECT_LIST_FAILURE, TOKEN_PRESERVATION_FAILURE, TYPE_CONSTRUCTOR_WORDS, + TYPE_DEPTH_FAILURE, UNSUPPORTED_DIALECT_FAILURE, UNSUPPORTED_SYNTAX_FAILURE, }; use crate::format::models::{FormatOptions, Formatted}; /// Format `sql`, or return why it is left as written. pub(crate) fn format_checked(sql: &str, options: &FormatOptions) -> Result { let dialect = options.dialect; - if dialect != Dialect::Snowflake { + if !FORMAT_GATED_DIALECTS.contains(&dialect) { return Err(UNSUPPORTED_DIALECT_FAILURE.to_string()); } let (lexemes, comments) = lex(sql, dialect)?; @@ -47,7 +47,7 @@ pub(crate) fn format_checked(sql: &str, options: &FormatOptions) -> Result Result, after: &AuthoredSql<'_>, binary_operators: &[bool], + dialect: Dialect, ) -> Result<(), String> { let pairs = before.lexemes.windows(2).zip(after.lexemes.windows(2)); for (index, (source, printed)) in pairs.enumerate() { @@ -25,7 +28,7 @@ pub(crate) fn verify_touching_tokens( { return Err(SEPARATION_FAILURE.to_string()); } - if !touched && touches && could_fuse(&source[0].raw, &source[1].raw) { + if !touched && touches && could_fuse(&source[0].raw, &source[1].raw, dialect) { return Err(JOIN_FAILURE.to_string()); } } @@ -89,14 +92,18 @@ fn operator_pieces(run: &str, offset: usize) -> Vec<(usize, usize)> { pieces } -/// Whether printing `previous` and `next` with no space between them could read as different tokens. -fn could_fuse(previous: &str, next: &str) -> bool { +/// Whether printing `previous` and `next` with no space between them could read as different tokens; DuckDB lexes `@` as an operator character, Snowflake as part of a stage name. +fn could_fuse(previous: &str, next: &str, dialect: Dialect) -> bool { let (Some(left), Some(right)) = (previous.chars().last(), next.chars().next()) else { return false; }; + let at_operator = dialect == Dialect::DuckDB; let word = |character: char| { - character.is_alphanumeric() || matches!(character, '_' | '$' | '\'' | '"' | '`' | '@') + character.is_alphanumeric() + || matches!(character, '_' | '$' | '\'' | '"' | '`') + || (character == '@' && !at_operator) }; - let operator = |character: char| OPERATOR_BYTES.contains(character); + let operator = + |character: char| OPERATOR_BYTES.contains(character) || (character == '@' && at_operator); (word(left) && word(right)) || (operator(left) && operator(right)) } diff --git a/crates/sqlquack-format/src/format/constants.rs b/crates/sqlquack-format/src/format/constants.rs index 6da828444..2bca7d813 100644 --- a/crates/sqlquack-format/src/format/constants.rs +++ b/crates/sqlquack-format/src/format/constants.rs @@ -21,7 +21,12 @@ pub(crate) const JOIN_FAILURE: &str = pub(crate) const SELECT_LIST_FAILURE: &str = "formatter could not find every select list, whose items without an alias it keeps as written"; pub(crate) const UNSUPPORTED_DIALECT_FAILURE: &str = - "formatter does not support this dialect yet; only Snowflake has a format gate"; + "formatter does not support this dialect yet; only Snowflake and DuckDB have a format gate"; +/// The dialects whose format corpus subset gates the formatter. +pub(crate) const FORMAT_GATED_DIALECTS: [sqlquack_syntax::Dialect; 2] = [ + sqlquack_syntax::Dialect::Snowflake, + sqlquack_syntax::Dialect::DuckDB, +]; pub(crate) const DEPTH_FAILURE: &str = "formatter does not lay out SQL whose parse tree is more than 512 levels deep"; /// The deepest parse tree the formatter walks; a deeper one, such as a chain of thousands of `+`, is refused before it can exhaust the stack. @@ -178,7 +183,7 @@ pub(crate) const PLACEHOLDER_WIDTH: usize = 20; /// The added width of a `::` cast, which the guide measures as `CAST(value AS type)`. pub(crate) const POSTFIX_CAST_WIDTH: usize = 10; /// Keywords the guide always separates from a following `(`. -pub(crate) const SPACED_BEFORE_PARENTHESIS: [&str; 42] = [ +pub(crate) const SPACED_BEFORE_PARENTHESIS: [&str; 44] = [ "ALL", "AND", "ANY", @@ -193,13 +198,12 @@ pub(crate) const SPACED_BEFORE_PARENTHESIS: [&str; 42] = [ "DISTINCT", "ELSE", "END", - "EXISTS", - "FILTER", "FROM", "GROUP", "HAVING", "IN", "JOIN", + "KEY", "LATERAL", "MATCH_CONDITION", "NOT", @@ -207,12 +211,15 @@ pub(crate) const SPACED_BEFORE_PARENTHESIS: [&str; 42] = [ "ON", "OR", "OVER", + "PERCENT", "QUALIFY", "REPEATABLE", + "RESERVOIR", "ROLLUP", "SAMPLE", "SEED", "SELECT", + "SETS", "SOME", "SYSTEM", "TABLESAMPLE", @@ -251,7 +258,7 @@ pub(crate) const TYPED_LITERALS: [&str; 7] = [ pub(crate) const TYPE_CONTINUATIONS: [&str; 6] = ["PRECISION", "VARYING", "WITH", "WITHOUT", "TIME", "ZONE"]; /// Calls that always put each argument on its own line. -pub(crate) const GROUPING_CALLS: [&str; 2] = ["ROLLUP", "CUBE"]; +pub(crate) const GROUPING_CALLS: [&str; 3] = ["ROLLUP", "CUBE", "SETS"]; /// Clause words of a Snowflake SEMANTIC_VIEW call, each starting its own line. pub(crate) const SEMANTIC_VIEW_CLAUSES: [&str; 4] = ["METRICS", "DIMENSIONS", "FACTS", "WHERE"]; /// Clause keywords that start a line of a SELECT. @@ -318,3 +325,13 @@ pub(crate) const SEMANTIC_VIEW_CALL: &str = "SEMANTIC_VIEW"; pub(crate) const OPEN_PARENTHESIS: &str = "("; /// A closing parenthesis. pub(crate) const CLOSE_PARENTHESIS: &str = ")"; +/// DuckDB's call that expands to columns, which a `*` before it unpacks. +pub(crate) const COLUMNS_CALL: &str = "COLUMNS"; +/// DuckDB's lambda keyword, whose parameters end at a `:`. +pub(crate) const LAMBDA_KEYWORD: &str = "LAMBDA"; +/// Keywords whose count a DuckDB `%` may follow as a percentage. +pub(crate) const SAMPLE_KEYWORDS: [&str; 3] = ["SAMPLE", "TABLESAMPLE", "LIMIT"]; +/// Keywords after which a `(` opens a parenthesised expression, not an argument list. +pub(crate) const EXPRESSION_LEADERS: [&str; 11] = [ + "AND", "OR", "NOT", "WHERE", "ON", "HAVING", "QUALIFY", "WHEN", "THEN", "ELSE", "SELECT", +]; diff --git a/crates/sqlquack-format/src/format/tests/test_format_corpus.rs b/crates/sqlquack-format/src/format/tests/test_format_corpus.rs index cdd5c0fa8..0be6a1c7b 100644 --- a/crates/sqlquack-format/src/format/tests/test_format_corpus.rs +++ b/crates/sqlquack-format/src/format/tests/test_format_corpus.rs @@ -7,106 +7,242 @@ const FORMAT_CORPUS: &str = include_str!("fixtures/format_corpus.json"); #[test] fn given_dialect_corpus_when_formatting_then_output_layout_and_refusals_are_stable() { - let test_cases = [test_types::FormatCorpusTestCase { - description: "Snowflake identity corpus", - dialect: "snowflake", - expected_formatted: 427, - expected_refused: 8, - departures: &[ - test_types::Departure { - sql: "WITH t (SELECT 1 AS c) SELECT c FROM t", - formatted: None, - reason: "Snowflake rejects a CTE body without AS, so SQLQuack's parser refuses it", - receipt: "sf.query.with.select.k8b29420d51a4", - }, - test_types::Departure { - sql: "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", - formatted: None, - reason: "Snowflake rejects a quoted date part in the FROM form of DATE_PART, so SQLQuack's parser refuses it", - receipt: "sf.functions.date-part.date-part-from.k10b598e047b5", - }, - test_types::Departure { - sql: "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", - formatted: Some( - "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data: level1 : level2 : level3::VARIANT\nFROM t", - ), - reason: "Snowflake names an unaliased select item by its text, spaces included, so the item keeps its spacing; SQLClean respaced it", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", - formatted: Some( - "SELECT\n RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", - formatted: Some( - "SELECT\n TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", - formatted: Some("SELECT\n ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])"), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", - formatted: Some( - "SELECT\n AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT {fn DAYNAME('2022-5-13')}", - formatted: Some("SELECT {fn DAYNAME('2022-5-13')}"), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT {fn LOG(5)}", - formatted: Some("SELECT {fn LOG(5)}"), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT {fn CEILING(5.3)}", - formatted: Some("SELECT {fn CEILING(5.3)}"), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", - formatted: Some( - "SELECT\n TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", - formatted: Some( - "SELECT\n SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - test_types::Departure { - sql: "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", - formatted: Some( - "SELECT\n SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", - ), - reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", - receipt: "cases/relational/unaliased_output_names.toml#unaliased_expressions", - }, - ], - }]; + let test_cases = [ + test_types::FormatCorpusTestCase { + description: "DuckDB identity corpus", + dialect: "duckdb", + expected_formatted: 189, + expected_refused: 0, + departures: &[ + test_types::Departure { + sql: "SELECT str[0:1]", + formatted: Some("SELECT str[0:1]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y : COALESCE(x, y, 0) + 1)", + formatted: Some( + "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x, y: COALESCE(x, y, 0) + 1)", + ), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x : x % 2 = 0), LAMBDA y : y * y)", + formatted: Some( + "SELECT LIST_TRANSFORM(LIST_FILTER([0, 1, 2, 3, 4, 5], LAMBDA x: x % 2 = 0), LAMBDA y: y * y)", + ), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x : COALESCE(x, 0) + 1)", + formatted: Some( + "SELECT LIST_TRANSFORM([5, NULL, 6], LAMBDA x: COALESCE(x, 0) + 1)", + ), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT LIST_TRANSFORM(nbr, LAMBDA x : x + 1) FROM article AS a", + formatted: Some( + "SELECT LIST_TRANSFORM(nbr, LAMBDA x: x + 1)\nFROM article AS a", + ), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT JSON_EXTRACT(c, '$[*].id')[0:2]", + formatted: Some("SELECT JSON_EXTRACT(c, '$[*].id')[0:2]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]", + formatted: Some("SELECT JSON_EXTRACT_STRING(c, '$[*].id')[0:2]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]", + formatted: Some("SELECT [1, 2, 3][1 + 1:LENGTH([1, 2, 3]) + -1]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT x::INT[3][3]", + formatted: Some("SELECT x::INT[3][3]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "SELECT ARRAY[[[1]]]::INT[1][1][1]", + formatted: Some("SELECT ARRAY[[[1]]]::INT[1][1][1]"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + test_types::Departure { + sql: "WITH t1 AS (FROM (FROM t2 SELECT foo1, foo2)) FROM t1", + formatted: Some( + "WITH t1 AS (\n FROM (\n FROM t2\n SELECT\n foo1,\n foo2\n )\n)\n\nFROM t1", + ), + reason: "SQLClean aligned against Polyglot's SELECT-first rewrite of the FROM-first query", + receipt: None, + }, + test_types::Departure { + sql: "SELECT e: 1 + 2, f: len('asdf'), s: (SELECT 42)", + formatted: Some( + "SELECT\n e:1 + 2,\n f:len('asdf'),\n s:(\n SELECT 42\n )", + ), + reason: "SQLClean aligned against Polyglot's path reading of the prefix alias and lost the line break after a comma", + receipt: None, + }, + test_types::Departure { + sql: "SELECT (@col) + 1 FROM t", + formatted: Some("SELECT\n (\n @col\n )\n + 1\nFROM t"), + reason: "SQLClean aligned against Polyglot's ABS() rewrite of `@`", + receipt: None, + }, + test_types::Departure { + sql: "SELECT (@-1) + 1", + formatted: Some("SELECT\n (\n @-1\n )\n + 1"), + reason: "SQLClean aligned against Polyglot's ABS() rewrite of `@`", + receipt: None, + }, + test_types::Departure { + sql: "SELECT file[:256] FROM GLOB('*')", + formatted: Some("SELECT file[:256]\nFROM GLOB('*')"), + reason: "SQLClean refused it: its Polyglot protection read a slice or lambda `:` as a path, or broke on a `::` array type", + receipt: None, + }, + ], + }, + test_types::FormatCorpusTestCase { + description: "Snowflake identity corpus", + dialect: "snowflake", + expected_formatted: 427, + expected_refused: 8, + departures: &[ + test_types::Departure { + sql: "WITH t (SELECT 1 AS c) SELECT c FROM t", + formatted: None, + reason: "Snowflake rejects a CTE body without AS, so SQLQuack's parser refuses it", + receipt: Some("sf.query.with.select.k8b29420d51a4"), + }, + test_types::Departure { + sql: "SELECT DATE_PART('month' FROM CAST('2024-04-08' AS DATE))", + formatted: None, + reason: "Snowflake rejects a quoted date part in the FROM form of DATE_PART, so SQLQuack's parser refuses it", + receipt: Some("sf.functions.date-part.date-part-from.k10b598e047b5"), + }, + test_types::Departure { + sql: "WITH t AS (SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data) SELECT data: level1 : level2 : level3::VARIANT FROM t", + formatted: Some( + "WITH t AS (\n SELECT PARSE_JSON('{\"level1\": {\"level2\": {\"level3\": \"value\"}}}') AS data\n)\n\nSELECT data: level1 : level2 : level3::VARIANT\nFROM t", + ), + reason: "Snowflake names an unaliased select item by its text, spaces included, so the item keeps its spacing; SQLClean respaced it", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + formatted: Some( + "SELECT\n RPAD(TO_BINARY('Hi', 'UTF8'), 10, TO_BINARY('_', 'UTF8'))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + formatted: Some( + "SELECT\n TIMESTAMP_FROM_PARTS(CAST('2024-05-09' AS DATE), CAST('14:30:45' AS TIME))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + formatted: Some( + "SELECT\n ARRAYS_ZIP([1, 2, 3], ['a', 'b', 'c'], [10, 20, 30])", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + formatted: Some( + "SELECT\n AI_CLASSIFY('text', ['travel', 'cooking'], OBJECT_CONSTRUCT('output_mode', 'multi'))", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT {fn DAYNAME('2022-5-13')}", + formatted: Some("SELECT {fn DAYNAME('2022-5-13')}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT {fn LOG(5)}", + formatted: Some("SELECT {fn LOG(5)}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT {fn CEILING(5.3)}", + formatted: Some("SELECT {fn CEILING(5.3)}"), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean broke a line inside its `{fn ...}` escape", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + formatted: Some( + "SELECT\n TIME_SLICE(CAST('2024-05-09 08:50:57.891' AS TIMESTAMP), 1, 'HOUR', 'start')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + formatted: Some( + "SELECT\n SEARCH(line, 'king', ANALYZER => 'UNICODE_ANALYZER', SEARCH_MODE => 'OR')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + test_types::Departure { + sql: "SELECT SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + formatted: Some( + "SELECT\n SEARCH(line, 'king', SEARCH_MODE => 'AND', ANALYZER => 'PATTERN_ANALYZER')", + ), + reason: "Snowflake names an unaliased select item by its text, line breaks included, so the item stays on one line; SQLClean split its arguments", + receipt: Some( + "cases/relational/unaliased_output_names.toml#unaliased_expressions", + ), + }, + ], + }, + ]; let corpus: Vec = serde_json::from_str(FORMAT_CORPUS).unwrap_or_default(); for test_case in test_cases { let outcome = check_dialect_corpus(&corpus, &test_case); diff --git a/crates/sqlquack-format/src/format/tests/test_format_sql.rs b/crates/sqlquack-format/src/format/tests/test_format_sql.rs index da3e5f0a7..408853f34 100644 --- a/crates/sqlquack-format/src/format/tests/test_format_sql.rs +++ b/crates/sqlquack-format/src/format/tests/test_format_sql.rs @@ -1,7 +1,7 @@ use sqlquack_syntax::Dialect; use crate::format::constants::{ - DEPTH_FAILURE, PARSE_FAILURE, SEPARATION_FAILURE, TYPE_DEPTH_FAILURE, + DEPTH_FAILURE, JOIN_FAILURE, PARSE_FAILURE, SEPARATION_FAILURE, TYPE_DEPTH_FAILURE, UNSUPPORTED_DIALECT_FAILURE, }; use crate::format::main::format_sql::format_sql; @@ -19,6 +19,15 @@ fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_re dialect: Dialect::Snowflake, expected_outcome: Ok("CREATE TABLE t(x object(type varchar, first int, key int))"), }, + test_types::FormatSqlTestCase { + description: "DuckDB struct field names spelled like keywords keep their case in a column type", + sql: "create table t (x struct(type varchar, first int, key int, date date))", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Ok( + "CREATE TABLE t(x struct(type varchar, first int, key int, date DATE))", + ), + }, test_types::FormatSqlTestCase { description: "an unquoted PUT file URI is not split at its slashes", sql: "PUT file:///data/x.csv @s1/p1/", @@ -209,11 +218,39 @@ fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_re expected_outcome: Ok("SELECT a\nFROM t -- one\n-- two"), }, test_types::FormatSqlTestCase { - description: "DuckDB is refused until it has a format gate", - sql: "SELECT a FROM t", + description: "a minus is not joined to a DuckDB `@`, which would read as the operator `-@`", + sql: "select - @a", line_width: None, dialect: Dialect::DuckDB, - expected_outcome: Err(UNSUPPORTED_DIALECT_FAILURE), + expected_outcome: Err(JOIN_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a plus is not joined to a DuckDB `@`", + sql: "select + @a", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Err(JOIN_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a DuckDB `@` is not joined to a plus sign", + sql: "select @ +1", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Err(JOIN_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a DuckDB `@` is not joined to a minus sign, which would read as the operator `@-`", + sql: "select @ -1", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Err(JOIN_FAILURE), + }, + test_types::FormatSqlTestCase { + description: "a DuckDB `@` is not joined to a minus before a column", + sql: "select @ - a", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Err(JOIN_FAILURE), }, test_types::FormatSqlTestCase { description: "PostgreSQL is refused until it has a format gate", @@ -249,11 +286,13 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou let test_cases = [ test_types::DeepSqlTestCase { description: "1,500 parentheses around a FROM source are refused by the parser's nesting guard", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT * FROM {}t{}", "(".repeat(1_500), ")".repeat(1_500)), expected_refusal: Some(PARSE_FAILURE), }, test_types::DeepSqlTestCase { description: "1,500 nested subqueries are refused by the parser's depth guard", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { format!( "{}SELECT 1{}", @@ -265,41 +304,55 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "1,500 nested NOTs are refused by the parser's depth guard", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT {}TRUE", "NOT ".repeat(1_500)), expected_refusal: Some(PARSE_FAILURE), }, test_types::DeepSqlTestCase { description: "a chain of 5,000 added terms is refused before its tree is walked", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT {}", vec!["1"; 5_000].join(" + ")), expected_refusal: Some(DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a chain of 513 added terms, one level past the limit, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT {}", vec!["1"; 513].join(" + ")), expected_refusal: Some(DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a cast to a type with 300 `[]` suffixes is refused before parsing", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT CAST(NULL AS INT{})", "[]".repeat(300)), expected_refusal: Some(TYPE_DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a cast to a type with 300 `[3]` suffixes is refused before parsing", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT CAST(NULL AS INT{})", "[3]".repeat(300)), expected_refusal: Some(TYPE_DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a column of a type with 65 `[]` suffixes, one past the limit, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("CREATE TABLE t (c INT{})", "[]".repeat(65)), expected_refusal: Some(TYPE_DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a column of a type with 64 `[]` suffixes, at the limit, is laid out", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("CREATE TABLE t (c INT{})", "[]".repeat(64)), expected_refusal: None, }, + test_types::DeepSqlTestCase { + description: "a DuckDB `::` cast to a type with 300 `[]` suffixes is refused before parsing", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], + sql: || format!("SELECT NULL::INT{}", "[]".repeat(300)), + expected_refusal: Some(TYPE_DEPTH_FAILURE), + }, test_types::DeepSqlTestCase { description: "a type alternating angle brackets with suffix runs, depth 182, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { format!( "SELECT CAST(NULL AS ARRAY>)", "[]".repeat(63)), expected_refusal: Some(TYPE_DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a cast to 2 angle brackets around 62 suffixes, depth 64, is laid out", + dialects: &[Dialect::Snowflake], sql: || format!("SELECT CAST(NULL AS ARRAY>)", "[]".repeat(62)), expected_refusal: None, }, test_types::DeepSqlTestCase { description: "a struct of 100 one-suffix fields stays shallow and is laid out", + dialects: &[Dialect::Snowflake], sql: || { format!( "SELECT CAST(NULL AS STRUCT<{}>)", @@ -334,6 +390,7 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a type alternating parenthesised constructors with suffix runs, depth 182, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { format!( "SELECT CAST(NULL AS ARRAY(ARRAY(INT{}{})", @@ -345,6 +402,7 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a type alternating quoted parenthesised constructors with suffix runs, depth 182, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { format!( "SELECT CAST(NULL AS \"ARRAY\"(\"ARRAY\"(INT{}{})", @@ -356,6 +414,7 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a column alternating quoted STRUCT<...> and ARRAY<...> with suffix runs is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { let run = "[]".repeat(40); format!("CREATE TABLE t (c \"STRUCT\"{run}>{run})") @@ -364,6 +423,7 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a column alternating STRUCT(...), MAP(...) and ARRAY<...> with suffix runs is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { let run = "[]".repeat(30); format!( @@ -374,16 +434,19 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a cast to 2 parenthesised constructors around 63 suffixes, depth 65, is refused", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT CAST(NULL AS ARRAY(ARRAY(INT{})))", "[]".repeat(63)), expected_refusal: Some(TYPE_DEPTH_FAILURE), }, test_types::DeepSqlTestCase { description: "a cast to 2 parenthesised constructors around 62 suffixes, depth 64, is laid out", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT CAST(NULL AS ARRAY(ARRAY(INT{})))", "[]".repeat(62)), expected_refusal: None, }, test_types::DeepSqlTestCase { description: "a cast to 300 nested ARRAY<...> types is refused before parsing", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || { format!( "SELECT CAST(NULL AS {}INT{})", @@ -395,19 +458,22 @@ fn given_deeply_nested_sql_when_formatting_then_it_is_refused_or_laid_out_withou }, test_types::DeepSqlTestCase { description: "a chain of 512 added terms, at the limit, is laid out", + dialects: &[Dialect::Snowflake, Dialect::DuckDB], sql: || format!("SELECT {}", vec!["1"; 512].join(" + ")), expected_refusal: None, }, ]; for test_case in test_cases { - let outcome = format_sql(&(test_case.sql)(), &FormatOptions::new(Dialect::Snowflake)); - assert_eq!( - outcome - .err() - .map(|refusal| refusal_start(&refusal.reason).to_string()), - test_case.expected_refusal.map(str::to_string), - "{}", - test_case.description - ); + for &dialect in test_case.dialects { + let outcome = format_sql(&(test_case.sql)(), &FormatOptions::new(dialect)); + assert_eq!( + outcome + .err() + .map(|refusal| refusal_start(&refusal.reason).to_string()), + test_case.expected_refusal.map(str::to_string), + "{dialect:?}: {}", + test_case.description + ); + } } } diff --git a/crates/sqlquack-format/src/format/tests/test_types.rs b/crates/sqlquack-format/src/format/tests/test_types.rs index 18eb4d6c0..2358ed40d 100644 --- a/crates/sqlquack-format/src/format/tests/test_types.rs +++ b/crates/sqlquack-format/src/format/tests/test_types.rs @@ -17,7 +17,7 @@ pub(super) struct Departure { pub(super) formatted: Option<&'static str>, pub(super) reason: &'static str, /// The recorded Snowflake evidence for SQLQuack's outcome: a `coverage/syntax` receipt or a `cases` case. - pub(super) receipt: &'static str, + pub(super) receipt: Option<&'static str>, } /// The formatted, refused and departing counts of one corpus dialect, and every check that failed. @@ -36,10 +36,12 @@ pub(super) struct EntryOutcome { pub(super) failures: Vec, } -/// A generated, deeply nested Snowflake query, and the refusal it gets, if any. +/// A generated, deeply nested query, and the refusal it gets in each gated dialect, if any. pub(super) struct DeepSqlTestCase { pub(super) description: &'static str, pub(super) sql: fn() -> String, + /// The dialects it is formatted in. + pub(super) dialects: &'static [sqlquack_syntax::Dialect], /// The start of the refusal reason; `None` when the query is laid out. pub(super) expected_refusal: Option<&'static str>, } diff --git a/docs/architecture.md b/docs/architecture.md index d12093816..7afca1c1d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,6 +37,7 @@ a parameter of every entry point; Snowflake is the only one implemented. | `sqlquack-duckdb` | DuckDB building blocks shared by rules, and DuckDB's own parse-and-bind check of every converted statement. | | `sqlquack-policy` | The policy file `sqlquack.toml`: schema, validation and type-pattern matching. | | `sqlquack-format` | The SQL formatter: lays SQL out over lines, changing only the whitespace between tokens and keyword case. A layout guide prints the authored tokens as a parse-tree pretty printer would; the output is refused unless it keeps every token, keeps comments attached and is idempotent. It builds on `sqlquack-syntax` alone. | +| `sqlquack-format-check` | Development only: proves that formatting DuckDB SQL leaves the output columns DuckDB binds it to unchanged, on the format corpus. | | `sqlquack-compare` | The strict comparator between a recorded result and a DuckDB replay or a fresh recording. | | `sqlquack-evidence` | Cases, case hashes, pending files, stored results, DuckDB replay, verification and audits. | | `sqlquack` | The `sqlquack` command line and the library entry points: the pipeline, the census, the index and the evidence commands. | diff --git a/fensu.toml b/fensu.toml index adf85af31..542ff4ef5 100644 --- a/fensu.toml +++ b/fensu.toml @@ -37,6 +37,7 @@ tooling = [ "crates/sqlquack-rules", "crates/sqlquack-duckdb", "crates/sqlquack-evidence", + "crates/sqlquack-format-check", "crates/sqlquack-compare", "crates/sqlquack-policy", ] From 71629f7967f29ddd1a3b33fe3b8dcf345d08549b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:46:59 +0000 Subject: [PATCH 20/22] feat(format): recase called names and lay out INSERT bodies as SQLClean does Called names are upper-cased like keywords, except SQLBuild placeholders, quoted names, relations and CTEs with column lists, DuckDB macro names, and Snowflake select items without an alias, whose text names the column. INSERT targets stay on the first line, angle-bracket type parameters stay touching, and struct field names, identifiers and binary operators are found in linear time. --- .../src/columns/tests/test_compare_columns.rs | 3 +- .../src/format/_helpers/guide.rs | 146 ++++++++++++++---- .../src/format/_helpers/identifiers.rs | 49 ++++-- .../src/format/_helpers/token_layout.rs | 17 +- .../sqlquack-format/src/format/constants.rs | 14 ++ .../src/format/tests/test_format_sql.rs | 29 +++- 6 files changed, 212 insertions(+), 46 deletions(-) diff --git a/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs b/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs index 699945ccf..9cf411056 100644 --- a/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs +++ b/crates/sqlquack-format-check/src/columns/tests/test_compare_columns.rs @@ -37,6 +37,7 @@ fn given_duckdb_queries_when_laid_out_then_duckdb_binds_the_same_output_columns( "select distinct A from (select 1 as a) where a=1 order by a", "select 1 x, 2 AS y, coalesce( a , 0 ) from (select null as a)", "select list_transform([1,2], x -> x+1), list_filter([1,2], lambda x: x>1)", + "select count(1), Coalesce(1, 2), upper('a'), Abs(-1), sum(range), list_value(1) from range(3)", "select interval 1 day, date '2024-01-01' + interval 1 day, current_date", "select t.a, t . a, t.* from (select 1 as a) as t", "select a+b as c, a+b c from (select 1 as a, 2 as b)", @@ -55,7 +56,7 @@ fn given_duckdb_queries_when_laid_out_then_duckdb_binds_the_same_output_columns( "select @ - a from (select 2 as a)", "select a - @b, @a, @(-1) from (select 1 as a, -2 as b)", ], - expected_bound: 42, + expected_bound: 44, expected_unbound: 4, expected_refused: 12, }, diff --git a/crates/sqlquack-format/src/format/_helpers/guide.rs b/crates/sqlquack-format/src/format/_helpers/guide.rs index b8a612477..9cdaefd8c 100644 --- a/crates/sqlquack-format/src/format/_helpers/guide.rs +++ b/crates/sqlquack-format/src/format/_helpers/guide.rs @@ -10,12 +10,13 @@ use crate::format::_helpers::lexemes::{ Lexeme, bracket_partners, ends_operand, is_keyword, is_unquoted_word, }; use crate::format::constants::{ - ALIAS_FOLLOWERS, CASE_BRANCHES, CLAUSE_KEYWORDS, COLUMNS_CALL, DUCKDB_INLINE_CALLS, - EXPRESSION_LEADERS, GROUPING_CALLS, GUIDE_INDENT, GUIDE_WIDTH, JOIN_PREFIXES, LAMBDA_KEYWORD, - PLACEHOLDER_WIDTH, POSTFIX_CAST_WIDTH, POSTGRES_INLINE_CALLS, SAMPLE_KEYWORDS, - SAMPLE_ROW_KEYWORD, SEMANTIC_VIEW_CALL, SEMANTIC_VIEW_CLAUSES, SET_OPERATOR_MODIFIERS, - SET_OPERATORS, SNOWFLAKE_INLINE_CALLS, SPACED_BEFORE_PARENTHESIS, STRUCT_TYPE_WORDS, - TYPE_CONTINUATIONS, TYPED_LITERALS, + ALIAS_FOLLOWERS, CALL_TARGET_KEYWORDS, CASE_BRANCHES, CLAUSE_KEYWORDS, COLUMNS_CALL, + CTE_BODY_PREFIXES, DUCKDB_INLINE_CALLS, EXPRESSION_LEADERS, GROUPING_CALLS, GUIDE_INDENT, + GUIDE_WIDTH, INSERT_BODIES, INSERT_KEYWORD, JOIN_PREFIXES, LAMBDA_KEYWORD, PARAMETERISED_TYPES, + PLACEHOLDER_NAME_PREFIX, PLACEHOLDER_WIDTH, POSTFIX_CAST_WIDTH, POSTGRES_INLINE_CALLS, + SAMPLE_KEYWORDS, SAMPLE_ROW_KEYWORD, SEMANTIC_VIEW_CALL, SEMANTIC_VIEW_CLAUSES, + SET_OPERATOR_MODIFIERS, SET_OPERATORS, SNOWFLAKE_INLINE_CALLS, SPACED_BEFORE_PARENTHESIS, + STRUCT_TYPE_WORDS, TYPE_CONTINUATIONS, TYPED_LITERALS, VALUES_KEYWORD, }; /// The guide's text and its tokens, with byte ranges in that text. @@ -54,6 +55,12 @@ struct Guide<'a> { enclosing: Vec>, /// Tokens of a Snowflake stage reference such as `@stage/path/file.csv`. in_stage: Vec, + /// Operators the parse tree reads as binary, such as a comparison's `<`. + binary_operators: &'a [bool], + /// The `<` and `>` around the parameters of a type such as `ARRAY`. + angles: Vec, + /// Tokens directly inside a struct type's field list, where a name keeps its case. + field_lists: Vec, /// The one-line spacing before each token. base: Vec<&'static str>, /// The laid-out whitespace before each token. @@ -104,6 +111,9 @@ impl<'a> Guide<'a> { partners: bracket_partners(lexemes), enclosing, in_stage: vec![false; count], + binary_operators: &names.binary_operators, + angles: vec![false; count], + field_lists: vec![false; count], base: vec![""; count], gaps: vec![String::new(); count], dropped: vec![false; count], @@ -112,6 +122,8 @@ impl<'a> Guide<'a> { protected: vec![false; count], }; guide.mark_stages(); + guide.mark_angles(); + guide.mark_field_lists(); for index in 1..count { guide.base[index] = guide.space(index); guide.gaps[index] = guide.base[index].to_string(); @@ -187,10 +199,31 @@ impl<'a> Guide<'a> { } } + /// Mark the `<` after a parameterised type name such as `ARRAY`, and the `>` that closes it. + fn mark_angles(&mut self) { + let mut open: usize = 0; + for index in 1..self.len() { + let kind = self.lexemes[index].kind; + if kind == TokenType::Lt + && !self.binary_operators[index] + && self.is_any(index - 1, &PARAMETERISED_TYPES) + { + self.angles[index] = true; + open += 1; + } else if kind == TokenType::Gt && open > 0 && !self.binary_operators[index] { + self.angles[index] = true; + open -= 1; + } + } + } + /// The spacing a one-line rendering puts before token `index`. fn space(&self, index: usize) -> &'static str { let (previous, next) = (&self.lexemes[index - 1], &self.lexemes[index]); - if self.in_stage[index] && self.in_stage[index - 1] { + if (self.in_stage[index] && self.in_stage[index - 1]) + || (self.angles[index - 1] && self.lexemes[index - 1].kind == TokenType::Lt) + || self.angles[index] + { return ""; } if self.in_stage[index - 1] && next.kind == TokenType::LParen { @@ -247,7 +280,9 @@ impl<'a> Guide<'a> { "" }; } - if next.kind == TokenType::LBracket && self.operand_ends(index - 1) { + if next.kind == TokenType::LBracket + && (self.operand_ends(index - 1) || self.angles[index - 1]) + { return ""; } " " @@ -559,11 +594,32 @@ impl<'a> Guide<'a> { self.lay_out_from_first(first, last, indent); } else if self.is(first, "WITH") { self.query(first, last, indent); + } else if self.is(first, INSERT_KEYWORD) { + self.insert(first, last, indent); } else { self.expr(first, last, indent); } } + /// Lay out an INSERT: its target on the first line, then its VALUES or query from a new line. + fn insert(&mut self, insert: usize, last: usize, indent: usize) { + let body = self + .tops(insert + 1, last) + .into_iter() + .find(|&index| self.is_any(index, &INSERT_BODIES)); + let Some(body) = body else { + self.expr(insert, last, indent); + return; + }; + self.expr(insert, body - 1, indent); + self.gaps[body] = newline(indent); + if self.is(body, VALUES_KEYWORD) { + self.expr(body, last, indent); + } else { + self.query(body, last, indent); + } + } + /// Lay out the CTEs of the WITH at `with`; return the token after the last CTE. fn common_table_expressions( &mut self, @@ -1064,7 +1120,7 @@ impl<'a> Guide<'a> { } } - /// The guide's spelling of token `index`: keywords in upper case, names as written. + /// The guide's spelling of token `index`: keywords and called names in upper case, other names as written. fn text(&self, index: usize) -> String { let lexeme = &self.lexemes[index]; if self.protected[index] @@ -1072,7 +1128,7 @@ impl<'a> Guide<'a> { || self.identifiers[index] || self.name_position(index) || self.field_name(index) - || !is_keyword(lexeme) + || !(is_keyword(lexeme) || self.called_name(index)) { lexeme.raw.clone() } else { @@ -1080,6 +1136,21 @@ impl<'a> Guide<'a> { } } + /// Whether token `index` is an unquoted name called with `(`, which folds to upper case as SQLBuild's query fingerprint folds it: not a placeholder, an INSERT or CREATE TABLE target, or a CTE name with a column list. + fn called_name(&self, index: usize) -> bool { + let raw = &self.lexemes[index].raw; + let Some(closer) = self.partners.get(index + 1).copied().flatten() else { + return false; + }; + self.kind(index + 1) == Some(TokenType::LParen) + && is_unquoted_word(raw) + && !raw.starts_with(PLACEHOLDER_NAME_PREFIX) + && !(index > 0 && self.is_any(index - 1, &CALL_TARGET_KEYWORDS)) + && !(self.is(closer + 1, "AS") + && (self.kind(closer + 2) == Some(TokenType::LParen) + || self.is_any(closer + 2, &CTE_BODY_PREFIXES))) + } + /// Whether token `index` names a field of a struct type such as `STRUCT(type VARCHAR)` or `STRUCT`; the tree keeps these names without spans, and a schema keeps their case. fn field_name(&self, index: usize) -> bool { let starts_field = index > 0 @@ -1094,31 +1165,46 @@ impl<'a> Guide<'a> { if !starts_field || !typed { return false; } - let mut angles: usize = 0; - let mut at = index; - while at > 0 { - at -= 1; - match self.lexemes[at].kind { - TokenType::Gt => angles += 1, - TokenType::GtGt => angles += 2, - TokenType::RParen | TokenType::RBracket => { - if let Some(opener) = self.partners[at].filter(|&opener| opener < at) { - at = opener; - } + self.field_lists[index] + } + + /// Mark every token whose innermost open bracket is a struct type's field list, such as the `(` of `STRUCT(a INT)` or the `<` of `STRUCT`, in one pass. + fn mark_field_lists(&mut self) { + let mut open: Vec<(TokenType, bool)> = Vec::new(); + for index in 0..self.len() { + self.field_lists[index] = open.last().is_some_and(|&(_, fields)| fields); + let kind = self.lexemes[index].kind; + let after_struct = index > 0 + && STRUCT_TYPE_WORDS + .iter() + .any(|word| self.lexemes[index - 1].raw.eq_ignore_ascii_case(word)); + match kind { + TokenType::LParen | TokenType::LBracket | TokenType::LBrace => { + open.push((kind, after_struct && kind == TokenType::LParen)); + } + TokenType::Lt if after_struct || self.angles[index] => { + open.push((kind, after_struct)); } - TokenType::Lt if angles > 0 => angles -= 1, - TokenType::Lt | TokenType::LParen | TokenType::LBracket => { - return angles == 0 - && self.lexemes[at].kind != TokenType::LBracket - && at > 0 - && STRUCT_TYPE_WORDS - .iter() - .any(|word| self.lexemes[at - 1].raw.eq_ignore_ascii_case(word)); + TokenType::RParen | TokenType::RBracket | TokenType::RBrace => { + while open + .pop() + .is_some_and(|(opener, _)| opener == TokenType::Lt) + {} + } + TokenType::Gt | TokenType::GtGt => { + let closes = if kind == TokenType::GtGt { 2 } else { 1 }; + for _ in 0..closes { + if open + .last() + .is_some_and(|&(opener, _)| opener == TokenType::Lt) + { + open.pop(); + } + } } _ => {} } } - false } fn render(&self) -> GuideLayout { diff --git a/crates/sqlquack-format/src/format/_helpers/identifiers.rs b/crates/sqlquack-format/src/format/_helpers/identifiers.rs index deeddd936..36993275c 100644 --- a/crates/sqlquack-format/src/format/_helpers/identifiers.rs +++ b/crates/sqlquack-format/src/format/_helpers/identifiers.rs @@ -77,15 +77,18 @@ pub(crate) fn parsed_names( converted.sort_unstable(); converted }; - let (identifiers, aliases) = (to_bytes(&ranges.identifiers), to_bytes(&ranges.aliases)); + let (identifiers, aliases) = ( + Covering::new(to_bytes(&ranges.identifiers)), + Covering::new(to_bytes(&ranges.aliases)), + ); Ok(ParsedNames { identifiers: lexemes .iter() - .map(|lexeme| covered(&identifiers, lexeme)) + .map(|lexeme| identifiers.covers(lexeme)) .collect(), aliases: lexemes .iter() - .map(|lexeme| covered(&aliases, lexeme)) + .map(|lexeme| aliases.covers(lexeme)) .collect(), binary_operators: binary_operators(&ranges, lexemes, &byte), selects: ranges.selects, @@ -108,9 +111,9 @@ fn binary_operators( let (start, end) = binary .span .map_or((0, usize::MAX), |(start, end)| (byte(start), byte(end))); - let inside: Vec = (0..lexemes.len()) - .filter(|&index| lexemes[index].start >= start && lexemes[index].end <= end) - .collect(); + let first = lexemes.partition_point(|lexeme| lexeme.start < start); + let last = lexemes.partition_point(|lexeme| lexeme.end <= end); + let inside: Vec = (first..last.max(first)).collect(); let found = match (binary.left_end.map(byte), binary.right_start.map(byte)) { (Some(left_end), _) => inside .iter() @@ -145,10 +148,36 @@ fn binary_operators( operators } -/// Whether one of the sorted byte `ranges` covers `lexeme`. -fn covered(ranges: &[(usize, usize)], lexeme: &Lexeme) -> bool { - let before = ranges.partition_point(|&(start, _)| start <= lexeme.start); - ranges[..before].iter().any(|&(_, end)| end >= lexeme.end) +/// Sorted byte ranges, with the furthest end among each range and those before it. +struct Covering { + ranges: Vec<(usize, usize)>, + furthest_ends: Vec, +} + +impl Covering { + fn new(ranges: Vec<(usize, usize)>) -> Self { + let furthest_ends: Vec = ranges + .iter() + .scan(0, |furthest, &(_, end)| { + *furthest = (*furthest).max(end); + Some(*furthest) + }) + .collect(); + Self { + ranges, + furthest_ends, + } + } + + /// Whether one of the ranges covers `lexeme`. + fn covers(&self, lexeme: &Lexeme) -> bool { + let before = self + .ranges + .partition_point(|&(start, _)| start <= lexeme.start); + before + .checked_sub(1) + .is_some_and(|last| self.furthest_ends[last] >= lexeme.end) + } } /// The identifier and alias ranges and the select lists of `value` and every node under it, walked without recursion so a deep tree cannot exhaust the stack. diff --git a/crates/sqlquack-format/src/format/_helpers/token_layout.rs b/crates/sqlquack-format/src/format/_helpers/token_layout.rs index 454c0f9d2..2c5f57bb2 100644 --- a/crates/sqlquack-format/src/format/_helpers/token_layout.rs +++ b/crates/sqlquack-format/src/format/_helpers/token_layout.rs @@ -86,7 +86,12 @@ impl Oracle { let generated: Vec = guide .tokens .iter() - .map(|token| Keyed::new(token, false)) + .enumerate() + .map(|(index, token)| { + let mut keyed = Keyed::new(token, false); + keyed.keyword |= is_called(&guide.tokens, index); + keyed + }) .collect(); Self { generated, @@ -236,7 +241,7 @@ pub(crate) fn verify_printed_tokens( } let allowed = recases.get(&index).is_some_and(|recase| { *recase == printed.raw - && is_keyword(printed) + && (is_keyword(printed) || is_called(after.lexemes, index)) && printed.raw == printed.raw.to_ascii_uppercase() && authored.raw.eq_ignore_ascii_case(&printed.raw) }); @@ -562,6 +567,14 @@ fn would_fuse(previous: &Keyed, next: &Keyed) -> bool { (word(left) && word(right)) || (operator(left) && operator(right)) } +/// Whether token `index` is an unquoted word followed by `(`, as a called name is. +fn is_called(lexemes: &[Lexeme], index: usize) -> bool { + is_unquoted_word(&lexemes[index].raw) + && lexemes + .get(index + 1) + .is_some_and(|next| next.kind == TokenType::LParen) +} + /// Whether the guide prints the aligned authored word as an upper-case keyword. fn confirms_keyword(authored: &Keyed, generated: &Keyed) -> bool { authored.recasable diff --git a/crates/sqlquack-format/src/format/constants.rs b/crates/sqlquack-format/src/format/constants.rs index 2bca7d813..6d9c4ff9d 100644 --- a/crates/sqlquack-format/src/format/constants.rs +++ b/crates/sqlquack-format/src/format/constants.rs @@ -35,6 +35,8 @@ pub(crate) const TYPE_DEPTH_FAILURE: &str = "formatter does not lay out data types nested more than 64 levels deep"; /// The deepest data type the formatter lets the parser build: a run of bracket suffixes such as `INT[][]`, nested type constructors, or both alternating. pub(crate) const MAX_TYPE_NESTING: usize = 64; +/// The types whose parameters are written in angle brackets, such as `ARRAY`. +pub(crate) const PARAMETERISED_TYPES: [&str; 5] = ["ARRAY", "STRUCT", "MAP", "OBJECT", "RANGE"]; /// The type names whose parameters are named fields, such as `STRUCT(a INT)`. pub(crate) const STRUCT_TYPE_WORDS: [&str; 5] = ["STRUCT", "ROW", "OBJECT", "UNION", "RECORD"]; /// The type names whose parameters are data types in angle brackets or parentheses, such as `ARRAY` or `STRUCT(a INT)`. @@ -261,6 +263,18 @@ pub(crate) const TYPE_CONTINUATIONS: [&str; 6] = pub(crate) const GROUPING_CALLS: [&str; 3] = ["ROLLUP", "CUBE", "SETS"]; /// Clause words of a Snowflake SEMANTIC_VIEW call, each starting its own line. pub(crate) const SEMANTIC_VIEW_CLAUSES: [&str; 4] = ["METRICS", "DIMENSIONS", "FACTS", "WHERE"]; +/// The start of a placeholder name, such as SQLBuild's `__ref(...)`, which keeps its case when called. +pub(crate) const PLACEHOLDER_NAME_PREFIX: &str = "__"; +/// Words after which a name with `(` is defined, not called: a table's column list or a macro's parameters. +pub(crate) const CALL_TARGET_KEYWORDS: [&str; 3] = ["INTO", "TABLE", "MACRO"]; +/// Words after a CTE's `AS` that start its body. +pub(crate) const CTE_BODY_PREFIXES: [&str; 2] = ["MATERIALIZED", "NOT"]; +/// The keyword that starts an INSERT statement. +pub(crate) const INSERT_KEYWORD: &str = "INSERT"; +/// The keyword of a VALUES list. +pub(crate) const VALUES_KEYWORD: &str = "VALUES"; +/// Words that start the rows an INSERT adds, each from a new line. +pub(crate) const INSERT_BODIES: [&str; 3] = ["VALUES", "SELECT", "WITH"]; /// Clause keywords that start a line of a SELECT. pub(crate) const CLAUSE_KEYWORDS: [&str; 8] = [ "FROM", "WHERE", "HAVING", "QUALIFY", "WINDOW", "LIMIT", "OFFSET", "FETCH", diff --git a/crates/sqlquack-format/src/format/tests/test_format_sql.rs b/crates/sqlquack-format/src/format/tests/test_format_sql.rs index 408853f34..c6d413ad5 100644 --- a/crates/sqlquack-format/src/format/tests/test_format_sql.rs +++ b/crates/sqlquack-format/src/format/tests/test_format_sql.rs @@ -17,7 +17,7 @@ fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_re sql: "create table t (x object(type varchar, first int, key int))", line_width: None, dialect: Dialect::Snowflake, - expected_outcome: Ok("CREATE TABLE t(x object(type varchar, first int, key int))"), + expected_outcome: Ok("CREATE TABLE t(x OBJECT(type varchar, first int, key int))"), }, test_types::FormatSqlTestCase { description: "DuckDB struct field names spelled like keywords keep their case in a column type", @@ -25,7 +25,30 @@ fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_re line_width: None, dialect: Dialect::DuckDB, expected_outcome: Ok( - "CREATE TABLE t(x struct(type varchar, first int, key int, date DATE))", + "CREATE TABLE t(x STRUCT(type varchar, first int, key int, date DATE))", + ), + }, + test_types::FormatSqlTestCase { + description: "a DuckDB macro name keeps its case, since DuckDB stores it as written", + sql: "create macro add_one(x) as x + 1", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Ok("CREATE macro add_one(x) AS x + 1"), + }, + test_types::FormatSqlTestCase { + description: "a temporary replacing DuckDB macro name keeps its case", + sql: "create or replace temp macro add_one(x) as x + 1", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Ok("CREATE OR REPLACE TEMP macro add_one(x) AS x + 1"), + }, + test_types::FormatSqlTestCase { + description: "a DuckDB table macro name keeps its case", + sql: "create macro first_rows(n) as table select * from t limit n", + line_width: None, + dialect: Dialect::DuckDB, + expected_outcome: Ok( + "CREATE macro first_rows(n) AS table SELECT\n *\nFROM t\nLIMIT n", ), }, test_types::FormatSqlTestCase { @@ -100,7 +123,7 @@ fn given_sql_whose_spacing_carries_meaning_when_formatting_then_it_is_kept_or_re sql: "select t.*, count(*) c from t where x=1", line_width: None, dialect: Dialect::Snowflake, - expected_outcome: Ok("SELECT\n t.*,\n count(*) c\nFROM t\nWHERE x = 1"), + expected_outcome: Ok("SELECT\n t.*,\n COUNT(*) c\nFROM t\nWHERE x = 1"), }, test_types::FormatSqlTestCase { description: "path and cast accessors stay touching", From 746d6e394bbd17a4a60800ea6bb5cb7be324bede Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:51:20 +0000 Subject: [PATCH 21/22] feat(lint): port the SQL lint engine Add the sqlquack-lint crate: SQLClean's lint rules, ported from its Polyglot engine onto SQLQuack's tokens and parse trees, so SQLClean and SQLBuild lint without Polyglot. Snowflake and DuckDB lint natively; dialects SQLQuack does not parse report lint as unavailable instead of failing. sqlquack-syntax gains two public entry points the engine needs: dialect_tokens, the token stream of a dialect, and parse_expressions, which reads already guarded tokens as expressions. The rule walkers ported from SQLClean keep its MIT notice in THIRD_PARTY_NOTICES. --- Cargo.lock | 17 + Cargo.toml | 3 + THIRD_PARTY_NOTICES | 2 + crates/sqlquack-lint/Cargo.toml | 20 + crates/sqlquack-lint/src/constants.rs | 25 + crates/sqlquack-lint/src/lib.rs | 18 + .../src/lint/_helpers/additional.rs | 1886 ++++++++++++++++ .../sqlquack-lint/src/lint/_helpers/engine.rs | 1959 +++++++++++++++++ .../src/lint/_helpers/inline_relations.rs | 223 ++ .../src/lint/_helpers/join_predicates.rs | 777 +++++++ crates/sqlquack-lint/src/lint/_helpers/mod.rs | 6 + .../src/lint/_helpers/parsing.rs | 102 + .../src/lint/_helpers/terminal_shape.rs | 373 ++++ crates/sqlquack-lint/src/lint/constants.rs | 117 + .../src/lint/main/batch_engine.rs | 19 + .../sqlquack-lint/src/lint/main/catalogue.rs | 6 + .../src/lint/main/cross_join_subject.rs | 15 + .../src/lint/main/default_rules.rs | 6 + crates/sqlquack-lint/src/lint/main/engine.rs | 5 + .../src/lint/main/fill_finding_path.rs | 9 + crates/sqlquack-lint/src/lint/main/lint.rs | 8 + crates/sqlquack-lint/src/lint/main/mod.rs | 9 + .../sqlquack-lint/src/lint/main/parse_sql.rs | 11 + .../src/lint/main/validate_dialect.rs | 9 + crates/sqlquack-lint/src/lint/mod.rs | 9 + crates/sqlquack-lint/src/lint/models.rs | 184 ++ crates/sqlquack-lint/src/lint/tests.rs | 18 + .../sqlquack-lint/src/lint/tests/helpers.rs | 104 + .../src/lint/tests/test_batch_lint.rs | 43 + .../src/lint/tests/test_cross_joins.rs | 284 +++ .../src/lint/tests/test_empty_fixture_lint.rs | 80 + .../src/lint/tests/test_lint_exemptions.rs | 173 ++ .../src/lint/tests/test_long_chains.rs | 75 + .../src/lint/tests/test_sql_lint.rs | 1923 ++++++++++++++++ .../src/lint/tests/test_types.rs | 154 ++ .../src/lint/tests/test_unused_table_alias.rs | 317 +++ .../src/quality/_helpers/cte_columns.rs | 480 ++++ .../src/quality/_helpers/json_parses.rs | 209 ++ .../src/quality/_helpers/keys.rs | 443 ++++ .../src/quality/_helpers/literals.rs | 67 + .../src/quality/_helpers/projection_spans.rs | 41 + .../src/quality/_helpers/ranking.rs | 228 ++ .../src/quality/_helpers/removal_safety.rs | 111 + .../src/quality/_helpers/safety.rs | 716 ++++++ .../src/quality/_helpers/setting_help.rs | 26 + .../src/quality/_helpers/syntax.rs | 126 ++ crates/sqlquack-lint/src/quality/constants.rs | 272 +++ .../src/quality/main/diagnostics.rs | 33 + crates/sqlquack-lint/src/quality/main/mod.rs | 1 + crates/sqlquack-lint/src/quality/mod.rs | 29 + crates/sqlquack-lint/src/quality/models.rs | 22 + crates/sqlquack-lint/src/quality/tests.rs | 12 + .../src/quality/tests/helpers.rs | 143 ++ .../src/quality/tests/test_json_parses.rs | 292 +++ .../src/quality/tests/test_quality_rules.rs | 472 ++++ .../src/quality/tests/test_quality_scale.rs | 43 + .../src/quality/tests/test_safety_rules.rs | 217 ++ .../src/quality/tests/test_types.rs | 49 + crates/sqlquack-lint/src/quality/types.rs | 6 + crates/sqlquack-lint/src/tree/_helpers/mod.rs | 3 + .../sqlquack-lint/src/tree/_helpers/scope.rs | 123 ++ crates/sqlquack-lint/src/tree/constants.rs | 4 + .../sqlquack-lint/src/tree/main/call_name.rs | 14 + .../src/tree/main/contains_aggregate.rs | 10 + .../src/tree/main/is_aggregate.rs | 93 + .../src/tree/main/is_row_number.rs | 11 + crates/sqlquack-lint/src/tree/main/mod.rs | 7 + .../src/tree/main/walk_in_scope.rs | 22 + crates/sqlquack-lint/src/tree/mod.rs | 5 + crates/sqlquack-syntax/src/lib.rs | 2 + crates/sqlquack-syntax/src/parser/main/mod.rs | 1 + .../src/parser/main/parse_expressions.rs | 18 + .../src/tokens/main/dialect_tokens.rs | 12 + crates/sqlquack-syntax/src/tokens/main/mod.rs | 1 + 74 files changed, 13353 insertions(+) create mode 100644 crates/sqlquack-lint/Cargo.toml create mode 100644 crates/sqlquack-lint/src/constants.rs create mode 100644 crates/sqlquack-lint/src/lib.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/additional.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/engine.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/inline_relations.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/join_predicates.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/mod.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/parsing.rs create mode 100644 crates/sqlquack-lint/src/lint/_helpers/terminal_shape.rs create mode 100644 crates/sqlquack-lint/src/lint/constants.rs create mode 100644 crates/sqlquack-lint/src/lint/main/batch_engine.rs create mode 100644 crates/sqlquack-lint/src/lint/main/catalogue.rs create mode 100644 crates/sqlquack-lint/src/lint/main/cross_join_subject.rs create mode 100644 crates/sqlquack-lint/src/lint/main/default_rules.rs create mode 100644 crates/sqlquack-lint/src/lint/main/engine.rs create mode 100644 crates/sqlquack-lint/src/lint/main/fill_finding_path.rs create mode 100644 crates/sqlquack-lint/src/lint/main/lint.rs create mode 100644 crates/sqlquack-lint/src/lint/main/mod.rs create mode 100644 crates/sqlquack-lint/src/lint/main/parse_sql.rs create mode 100644 crates/sqlquack-lint/src/lint/main/validate_dialect.rs create mode 100644 crates/sqlquack-lint/src/lint/mod.rs create mode 100644 crates/sqlquack-lint/src/lint/models.rs create mode 100644 crates/sqlquack-lint/src/lint/tests.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/helpers.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_batch_lint.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_cross_joins.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_empty_fixture_lint.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_lint_exemptions.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_long_chains.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_types.rs create mode 100644 crates/sqlquack-lint/src/lint/tests/test_unused_table_alias.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/cte_columns.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/json_parses.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/keys.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/literals.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/projection_spans.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/ranking.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/removal_safety.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/safety.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/setting_help.rs create mode 100644 crates/sqlquack-lint/src/quality/_helpers/syntax.rs create mode 100644 crates/sqlquack-lint/src/quality/constants.rs create mode 100644 crates/sqlquack-lint/src/quality/main/diagnostics.rs create mode 100644 crates/sqlquack-lint/src/quality/main/mod.rs create mode 100644 crates/sqlquack-lint/src/quality/mod.rs create mode 100644 crates/sqlquack-lint/src/quality/models.rs create mode 100644 crates/sqlquack-lint/src/quality/tests.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/helpers.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/test_json_parses.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/test_quality_rules.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/test_quality_scale.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/test_safety_rules.rs create mode 100644 crates/sqlquack-lint/src/quality/tests/test_types.rs create mode 100644 crates/sqlquack-lint/src/quality/types.rs create mode 100644 crates/sqlquack-lint/src/tree/_helpers/mod.rs create mode 100644 crates/sqlquack-lint/src/tree/_helpers/scope.rs create mode 100644 crates/sqlquack-lint/src/tree/constants.rs create mode 100644 crates/sqlquack-lint/src/tree/main/call_name.rs create mode 100644 crates/sqlquack-lint/src/tree/main/contains_aggregate.rs create mode 100644 crates/sqlquack-lint/src/tree/main/is_aggregate.rs create mode 100644 crates/sqlquack-lint/src/tree/main/is_row_number.rs create mode 100644 crates/sqlquack-lint/src/tree/main/mod.rs create mode 100644 crates/sqlquack-lint/src/tree/main/walk_in_scope.rs create mode 100644 crates/sqlquack-lint/src/tree/mod.rs create mode 100644 crates/sqlquack-syntax/src/parser/main/parse_expressions.rs create mode 100644 crates/sqlquack-syntax/src/tokens/main/dialect_tokens.rs diff --git a/Cargo.lock b/Cargo.lock index 6a1e4c19a..025499dae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1174,6 +1174,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1435,6 +1441,17 @@ dependencies = [ "sqlquack-syntax", ] +[[package]] +name = "sqlquack-lint" +version = "0.2.0" +dependencies = [ + "rayon", + "rustc-hash", + "serde", + "serde_json", + "sqlquack-syntax", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index cfc271a01..ae7ba1c69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/sqlquack-syntax-postgres", "crates/sqlquack-format", "crates/sqlquack-format-check", + "crates/sqlquack-lint", "crates/sqlquack-syntax-bigquery", "crates/sqlquack-fingerprint", "crates/sqlquack-python", @@ -51,6 +52,7 @@ polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "7d5 proc-macro2 = "1" quote = "1" rayon = "1" +rustc-hash = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" # The Python module; abi3, so one build loads into every CPython from 3.12. @@ -63,6 +65,7 @@ sqlquack-duckdb = { path = "crates/sqlquack-duckdb", default-features = false } sqlquack-evidence = { path = "crates/sqlquack-evidence" } sqlquack-format = { path = "crates/sqlquack-format" } sqlquack-fingerprint = { path = "crates/sqlquack-fingerprint" } +sqlquack-lint = { path = "crates/sqlquack-lint" } sqlquack-policy = { path = "crates/sqlquack-policy" } sqlquack-rules = { path = "crates/sqlquack-rules" } sqlquack-semantics = { path = "crates/sqlquack-semantics" } diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 796addf9c..287055ddf 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -63,6 +63,8 @@ SQLQuack's DuckDB result-type rules, in crates/sqlquack-semantics/src/lineage/_h and duckdb_types.rs, are ported from polyglot's engine type rules, and the fixture crates/sqlquack-semantics/src/lineage/tests/fixtures/duckdb_engine_types.json is polyglot's tests/fixtures/engine_types/duckdb.json, the result types DuckDB bound for its synthetic cases. +The lint Rules' scope walk and aggregate classifier, in crates/sqlquack-lint/src/tree, are ported +from polyglot's scope.rs and traversal.rs. polyglot is distributed under the following licence: diff --git a/crates/sqlquack-lint/Cargo.toml b/crates/sqlquack-lint/Cargo.toml new file mode 100644 index 000000000..4b6453c97 --- /dev/null +++ b/crates/sqlquack-lint/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sqlquack-lint" +version.workspace = true +description = "SQL lint Rules over SQLQuack's tokens and syntax trees, ported from SQLClean." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +rayon.workspace = true +rustc-hash.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlquack-syntax.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-lint/src/constants.rs b/crates/sqlquack-lint/src/constants.rs new file mode 100644 index 000000000..068c43da2 --- /dev/null +++ b/crates/sqlquack-lint/src/constants.rs @@ -0,0 +1,25 @@ +//! Constants shared by the engine modules, and the placeholders hosts write for template code. + +pub(crate) const SQL_WILDCARD: &str = "*"; +pub(crate) const OPEN_PAREN: &str = "("; +pub(crate) const CLOSE_PAREN: &str = ")"; +pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; +pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; +/// The indent of each line of a configuration snippet printed in remediation text. +pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; +/// The configuration file remediation text names when the request does not name one. +pub const DEFAULT_CONFIG_FILE: &str = "sqlclean.toml"; +/// The identifier prefix a host writes in place of each template interpolation. +pub const INTERPOLATION_PLACEHOLDER_PREFIX: &str = "__sqlclean_interpolation_"; +/// The identifier prefix a host writes in place of a SQL macro parameter. +pub const MACRO_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_macro_parameter_"; +/// The prefix of a relation parameter placeholder, written `__`. +pub const RELATION_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_relation_parameter_"; +/// The identifier prefix a host writes in place of a value parameter. +pub const VALUE_PARAMETER_PLACEHOLDER_PREFIX: &str = "__sqlclean_value_parameter_"; +/// Placeholders standing for names the host generates, which Rules never judge as authored names. +pub(crate) const GENERATED_PLACEHOLDER_PREFIXES: [&str; 3] = [ + INTERPOLATION_PLACEHOLDER_PREFIX, + RELATION_PARAMETER_PLACEHOLDER_PREFIX, + VALUE_PARAMETER_PLACEHOLDER_PREFIX, +]; diff --git a/crates/sqlquack-lint/src/lib.rs b/crates/sqlquack-lint/src/lib.rs new file mode 100644 index 000000000..0234da54d --- /dev/null +++ b/crates/sqlquack-lint/src/lib.rs @@ -0,0 +1,18 @@ +//! SQL lint Rules over SQLQuack's tokens and syntax trees, ported from SQLClean's lint engine. + +#![forbid(unsafe_code)] + +pub mod constants; +pub mod lint; +pub mod quality; +pub(crate) mod tree; + +pub use crate::lint::main::batch_engine::lint_batch_json; +pub use crate::lint::main::catalogue::catalogue; +pub use crate::lint::main::cross_join_subject::cross_join_subject; +pub use crate::lint::main::default_rules::default_rules; +pub use crate::lint::main::engine::lint_json; +pub use crate::lint::main::fill_finding_path::fill_finding_path; +pub use crate::lint::main::lint::lint; +pub use crate::lint::main::parse_sql::parse_sql; +pub use crate::lint::main::validate_dialect::validate_dialect; diff --git a/crates/sqlquack-lint/src/lint/_helpers/additional.rs b/crates/sqlquack-lint/src/lint/_helpers/additional.rs new file mode 100644 index 000000000..563fb7141 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/additional.rs @@ -0,0 +1,1886 @@ +use std::collections::HashSet; + +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; + +use crate::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; +use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; +use crate::constants::{ + GENERATED_PLACEHOLDER_PREFIXES, MACRO_PARAMETER_PLACEHOLDER_PREFIX, + RELATION_PARAMETER_PLACEHOLDER_PREFIX, +}; +use crate::lint::_helpers::engine::{ + direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, + significant_before, token_depths, +}; +use crate::lint::constants::RELATION_MODIFIER_KEYWORDS; +use crate::lint::models::{ + AdditionalFactOptions, AdditionalQueryFacts, FixtureHarness, SqlMacroOutput, +}; +use rustc_hash::{FxHashMap, FxHashSet}; + +const COUNT_ONE_LITERAL: &str = "1"; +const QUALIFIED_REFERENCE_LENGTH: usize = 3; +const QUALIFIER_PAIR_LENGTH: usize = 2; +const IMPLICIT_ALIAS_MINIMUM_LENGTH: usize = 2; + +struct QuerySlice<'a> { + tokens: &'a [Token], + depths: &'a [usize], + direct: &'a [usize], + start: usize, + end: usize, + depth: usize, +} + +struct ReferenceFactContext<'a> { + query: &'a QuerySlice<'a>, + outer_relations: &'a FxHashSet, + external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], + query_start: usize, + query_end: usize, +} + +struct ReferenceCandidateContext<'a> { + tokens: &'a [Token], + references: &'a [usize], + external_identifiers: &'a HashSet, + projection_aliases: &'a FxHashMap, +} + +struct SelectFactContext<'a> { + tokens: &'a [Token], + depths: &'a [usize], + external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], + dependency_identifiers: &'a HashSet, + allows_ceremonial_select: bool, + allows_dynamic_output_star: bool, + fixture_harness: Option<&'a FixtureHarness>, +} + +struct ProjectedStarContext<'a> { + tokens: &'a [Token], + depths: &'a [usize], + direct: &'a [usize], + query_end: usize, +} + +pub(super) fn collect_additional_facts( + tokens: &[Token], + options: &AdditionalFactOptions<'_>, +) -> AdditionalQueryFacts { + let mut facts = AdditionalQueryFacts::default(); + let depths = token_depths(tokens); + let significant: Vec = (0..tokens.len()) + .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) + .collect(); + for (position, &index) in significant.iter().enumerate() { + let token = &tokens[index]; + if token.token_type == TokenType::Union + && significant.get(position + 1).is_none_or(|&next| { + !matches!( + tokens[next].token_type, + TokenType::All | TokenType::Distinct | TokenType::By + ) + }) + { + facts.bare_unions.push(token.span); + } + if token.token_type == TokenType::Else + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].token_type == TokenType::Null) + { + let next = significant[position + 1]; + facts.redundant_else_nulls.push(Span { + start: token.span.start, + end: tokens[next].span.end, + line: token.span.line, + column: token.span.column, + }); + } + if token.token_type == TokenType::Distinct + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) + && let Some(span) = + parenthesized_distinct_span(tokens, &depths, significant[position + 1]) + { + facts.parenthesized_distincts.push(span); + } + if token.token_type == TokenType::Semicolon + && significant + .get(position.wrapping_sub(1)) + .is_some_and(|&previous| tokens[previous].token_type == TokenType::Semicolon) + { + facts.consecutive_semicolons.push(token.span); + } + if is_identifier(token) + && let Some((&as_index, &alias_index)) = significant + .get(position + 1) + .zip(significant.get(position + 2)) + && tokens[as_index].token_type == TokenType::As + && is_identifier(&tokens[alias_index]) + && token.token_type == tokens[alias_index].token_type + && token.text == tokens[alias_index].text + && is_simple_projection_reference(tokens, &depths, &significant, position) + { + facts.redundant_self_aliases.push(Span { + start: token.span.end, + end: tokens[alias_index].span.end, + line: tokens[as_index].span.line, + column: tokens[as_index].span.column, + }); + } + if token.text.eq_ignore_ascii_case("COUNT") + && let Some(span) = count_one_span(tokens, &significant, position) + { + facts.count_one_literals.push(span); + } + if token.text.eq_ignore_ascii_case("ROW_NUMBER") + && let Some(span) = unstable_row_number_span(tokens, &depths, index) + { + facts.unstable_row_numbers.push(span); + } + if token.token_type == TokenType::Not + && let Some(span) = null_not_in_span(tokens, &depths, &significant, position) + { + facts.null_not_in_predicates.push(span); + } + if token.token_type == TokenType::Case + && let Some(span) = simple_boolean_case_span(tokens, &significant, position) + { + facts.simple_boolean_cases.push(span); + } + if token.token_type == TokenType::Else + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].token_type == TokenType::Case) + { + facts + .nested_else_cases + .push(tokens[significant[position + 1]].span); + } + if position + 2 < significant.len() + && constant_predicate(tokens, &significant[position..position + 3]) + { + facts + .constant_predicates + .push(tokens[significant[position + 1]].span); + } + } + let select_facts = collect_select_additional_facts(&SelectFactContext { + tokens, + depths: &depths, + external_identifiers: options.external_identifiers, + sql_macro_outputs: options.sql_macro_outputs, + dependency_identifiers: options.dependency_identifiers, + allows_ceremonial_select: options.allows_ceremonial_select, + allows_dynamic_output_star: options.allows_dynamic_output_star, + fixture_harness: options.fixture_harness, + }); + facts.duplicate_output_aliases = select_facts.duplicate_output_aliases; + facts.unaliased_calculations = select_facts.unaliased_calculations; + facts.projected_stars = select_facts.projected_stars; + facts.duplicate_table_aliases = select_facts.duplicate_table_aliases; + facts.unused_table_aliases = select_facts.unused_table_aliases; + facts.null_rejected_left_joins = select_facts.null_rejected_left_joins; + facts.implicit_inner_joins = select_facts.implicit_inner_joins; + facts.unqualified_multi_source_columns = select_facts.unqualified_multi_source_columns; + facts.inconsistent_single_source_qualification = + select_facts.inconsistent_single_source_qualification; + facts.unknown_relation_qualifiers = select_facts.unknown_relation_qualifiers; + facts.unused_joined_relations = select_facts.unused_joined_relations; + facts.mixed_group_order_references = select_facts.mixed_group_order_references; + facts.ambiguous_order_directions = select_facts.ambiguous_order_directions; + facts.set_arity_mismatches = set_arity_mismatch_spans(tokens, &depths); + facts +} + +fn is_simple_projection_reference( + tokens: &[Token], + depths: &[usize], + significant: &[usize], + position: usize, +) -> bool { + let index = significant[position]; + let Some(boundary) = (0..index).rev().find(|&candidate| { + depths[candidate] == depths[index] + && (tokens[candidate].token_type == TokenType::Select + || is_query_from(tokens, candidate) + || tokens[candidate].token_type == TokenType::Join) + }) else { + return false; + }; + if tokens[boundary].token_type != TokenType::Select { + return false; + } + let item_boundary = significant[..position] + .iter() + .rposition(|&candidate| { + candidate > boundary + && depths[candidate] == depths[index] + && tokens[candidate].token_type == TokenType::Comma + }) + .map_or(boundary, |boundary_position| significant[boundary_position]); + let prefix: Vec = significant[..position] + .iter() + .copied() + .filter(|&candidate| candidate > item_boundary && depths[candidate] == depths[index]) + .collect(); + if is_bare_star_prefix(tokens, &prefix) { + return true; + } + let mut qualifier_start = prefix.len(); + while qualifier_start >= QUALIFIER_PAIR_LENGTH + && tokens[prefix[qualifier_start - 1]].token_type == TokenType::Dot + && is_identifier(&tokens[prefix[qualifier_start - 2]]) + { + qualifier_start -= 2; + } + qualifier_start < prefix.len() && is_bare_star_prefix(tokens, &prefix[..qualifier_start]) +} + +fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> AdditionalQueryFacts { + let tokens = context.tokens; + let depths = context.depths; + let mut facts = AdditionalQueryFacts::default(); + for (select_index, token) in tokens.iter().enumerate() { + if token.token_type != TokenType::Select { + continue; + } + let depth = depths[select_index]; + let end = query_end(tokens, depths, select_index, depth); + let direct: Vec = direct_indices(depths, select_index + 1, end, depth) + .into_iter() + .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) + .collect(); + let from_position = direct + .iter() + .position(|&index| is_query_from(tokens, index)) + .unwrap_or(direct.len()); + facts + .duplicate_output_aliases + .extend(duplicate_alias_spans(tokens, &direct[..from_position])); + if !is_exists_subquery(tokens, select_index) + && !is_set_continuation_select(tokens, select_index) + && !is_scalar_subquery(tokens, depths, select_index) + && !has_enclosing_cte_column_list(tokens, depths, select_index) + && !is_ceremonial_select( + tokens, + &direct[..from_position], + context.allows_ceremonial_select, + ) + { + facts + .unaliased_calculations + .extend(unaliased_calculation_spans( + tokens, + &direct[..from_position], + )); + } + if !is_dependency_import_select(context, select_index, &direct, from_position) + && !is_compiler_proven_dynamic_output_select( + context, + select_index, + &direct, + from_position, + ) + && !is_canonical_empty_input_fixture(context, select_index, end) + && !has_irreducible_projected_star( + &ProjectedStarContext { + tokens, + depths, + direct: &direct, + query_end: end, + }, + from_position, + ) + { + facts + .projected_stars + .extend(projected_star_spans(tokens, &direct[..from_position])); + } + if from_position < direct.len() { + let clause_end = direct[from_position + 1..] + .iter() + .position(|&index| is_after_relation_clause(tokens[index].token_type)) + .map_or(direct.len(), |offset| from_position + 1 + offset); + facts.duplicate_table_aliases.extend(duplicate_alias_spans( + tokens, + &direct[from_position + 1..clause_end], + )); + let query_slice = QuerySlice { + tokens, + depths, + direct: &direct[from_position + 1..clause_end], + start: select_index, + end, + depth, + }; + facts + .unused_table_aliases + .extend(unused_alias_spans(&query_slice)); + facts + .null_rejected_left_joins + .extend(null_rejected_left_join_spans(tokens, &direct, clause_end)); + facts.implicit_inner_joins.extend(implicit_inner_join_spans( + tokens, + &direct[from_position + 1..clause_end], + )); + let reference_slice = QuerySlice { + tokens, + depths, + direct: &direct, + start: from_position, + end: clause_end, + depth, + }; + let outer_relations = outer_relation_names(tokens, depths, select_index, depth); + let reference_context = ReferenceFactContext { + query: &reference_slice, + outer_relations: &outer_relations, + external_identifiers: context.external_identifiers, + sql_macro_outputs: context.sql_macro_outputs, + query_start: select_index, + query_end: end, + }; + let reference_facts = collect_reference_facts(&reference_context); + facts + .unqualified_multi_source_columns + .extend(reference_facts.unqualified_multi_source_columns); + facts + .inconsistent_single_source_qualification + .extend(reference_facts.inconsistent_single_source_qualification); + facts + .unknown_relation_qualifiers + .extend(reference_facts.unknown_relation_qualifiers); + facts + .unused_joined_relations + .extend(reference_facts.unused_joined_relations); + } + facts + .mixed_group_order_references + .extend(mixed_reference_clause_spans(tokens, &direct)); + facts + .ambiguous_order_directions + .extend(ambiguous_order_direction_spans(tokens, &direct)); + } + facts +} + +fn is_canonical_empty_input_fixture( + context: &SelectFactContext<'_>, + select_index: usize, + query_end: usize, +) -> bool { + let Some(harness) = context.fixture_harness else { + return false; + }; + let Some(empty_relation) = harness.empty_relation.as_deref() else { + return false; + }; + let significant_query: Vec<&str> = context.tokens[select_index..query_end] + .iter() + .filter(|token| !is_layout(token) && !is_comment(token)) + .map(|token| token.text.as_str()) + .collect(); + if significant_query.len() != EMPTY_FIXTURE_QUERY_TOKEN_COUNT + || !significant_query[0].eq_ignore_ascii_case("select") + || significant_query[1] != SQL_WILDCARD + || !significant_query[2].eq_ignore_ascii_case("from") + || !significant_query[3].eq_ignore_ascii_case(empty_relation) + || significant_query[4] != OPEN_PAREN + || significant_query[5] != CLOSE_PAREN + { + return false; + } + let previous: Vec<&Token> = context.tokens[..select_index] + .iter() + .filter(|token| !is_layout(token) && !is_comment(token)) + .rev() + .take(ENCLOSING_CTE_TOKEN_COUNT) + .collect(); + if previous.len() != ENCLOSING_CTE_TOKEN_COUNT + || previous[0].token_type != TokenType::LParen + || !previous[1].text.eq_ignore_ascii_case("as") + { + return false; + } + let cte_name = previous[2].text.trim_matches(['"', '`']); + harness + .input_cte_prefixes + .iter() + .any(|prefix| cte_name.starts_with(prefix.as_str())) +} + +fn is_compiler_proven_dynamic_output_select( + context: &SelectFactContext<'_>, + select_index: usize, + direct: &[usize], + from_position: usize, +) -> bool { + if !context.allows_dynamic_output_star + || context.depths[select_index] != 0 + || direct[..from_position].len() != 1 + || context.tokens[direct[0]].token_type != TokenType::Star + { + return false; + } + !context.tokens[select_index + 1..] + .iter() + .zip(&context.depths[select_index + 1..]) + .any(|(token, depth)| token.token_type == TokenType::Select && *depth == 0) +} + +fn has_irreducible_projected_star( + context: &ProjectedStarContext<'_>, + from_position: usize, +) -> bool { + let ProjectedStarContext { + tokens, + depths, + direct, + query_end, + } = context; + let projection = &direct[..from_position]; + if projection.len() != 1 || tokens[projection[0]].token_type != TokenType::Star { + return false; + } + + let Some(relation_start) = from_position + .checked_add(1) + .filter(|&start| start < direct.len()) + else { + return false; + }; + let relation_end = direct[relation_start..] + .iter() + .position(|&index| is_after_relation_clause(tokens[index].token_type)) + .map_or(direct.len(), |offset| relation_start + offset); + let relation = &direct[relation_start..relation_end]; + if relation + .iter() + .any(|&index| matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma)) + { + return false; + } + if is_lone_audit_parameter_relation(tokens, relation) { + return true; + } + + let mut pivots = relation + .iter() + .copied() + .filter(|&index| tokens[index].text.eq_ignore_ascii_case("PIVOT")); + let Some(pivot_index) = pivots.next() else { + return false; + }; + if pivots.next().is_some() { + return false; + } + dynamic_pivot_uses_any(tokens, depths, pivot_index, *query_end) +} + +fn is_lone_audit_parameter_relation(tokens: &[Token], relation: &[usize]) -> bool { + let Some(&relation_index) = relation.first() else { + return false; + }; + let normalized = tokens[relation_index].text.to_ascii_lowercase(); + let Some(index) = normalized + .strip_prefix(RELATION_PARAMETER_PLACEHOLDER_PREFIX) + .and_then(|value| value.strip_suffix("__")) + else { + return false; + }; + if index.is_empty() || !index.chars().all(|character| character.is_ascii_digit()) { + return false; + } + match relation.get(1..) { + Some([]) => true, + Some([alias]) => is_identifier(&tokens[*alias]), + Some([as_index, alias]) => { + tokens[*as_index].token_type == TokenType::As && is_identifier(&tokens[*alias]) + } + _ => false, + } +} + +fn dynamic_pivot_uses_any( + tokens: &[Token], + depths: &[usize], + pivot_index: usize, + query_end: usize, +) -> bool { + let Some(pivot_open) = significant_after(tokens, pivot_index) else { + return false; + }; + if tokens[pivot_open].token_type != TokenType::LParen { + return false; + } + let pivot_depth = depths[pivot_open]; + let Some(pivot_close) = (pivot_open + 1..query_end).find(|&index| { + depths[index] == pivot_depth + 1 && tokens[index].token_type == TokenType::RParen + }) else { + return false; + }; + (pivot_open + 1..pivot_close).any(|in_index| { + depths[in_index] == pivot_depth + 1 + && tokens[in_index].token_type == TokenType::In + && significant_after(tokens, in_index).is_some_and(|any_open| { + any_open < pivot_close + && tokens[any_open].token_type == TokenType::LParen + && significant_after(tokens, any_open).is_some_and(|any_index| { + any_index < pivot_close && tokens[any_index].token_type == TokenType::Any + }) + }) + }) +} + +fn has_enclosing_cte_column_list(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { + let select_depth = depths[select_index]; + let Some(body_open) = (0..select_index).rev().find(|&index| { + tokens[index].token_type == TokenType::LParen + && depths[index].saturating_add(1) == select_depth + }) else { + return false; + }; + let Some(as_index) = significant_before(tokens, body_open) else { + return false; + }; + if tokens[as_index].token_type != TokenType::As { + return false; + } + significant_before(tokens, as_index) + .is_some_and(|index| tokens[index].token_type == TokenType::RParen) +} + +fn is_dependency_import_select( + context: &SelectFactContext<'_>, + select_index: usize, + direct: &[usize], + from_position: usize, +) -> bool { + let tokens = context.tokens; + let depths = context.depths; + if from_position != 1 + || tokens[direct[0]].token_type != TokenType::Star + || !is_top_level_cte_body(tokens, depths, select_index) + { + return false; + } + if direct.len() <= from_position + 1 || has_cte_body_set_operation(tokens, depths, select_index) + { + return false; + } + let relation = &direct[from_position + 1..]; + let Some(&relation_index) = relation.first() else { + return false; + }; + if !context + .dependency_identifiers + .contains(&tokens[relation_index].text.to_ascii_lowercase()) + { + return false; + } + match relation { + [_] => true, + [_, alias] => is_identifier(&tokens[*alias]), + [_, explicit_as, alias] => { + tokens[*explicit_as].token_type == TokenType::As && is_identifier(&tokens[*alias]) + } + _ => false, + } +} + +fn has_cte_body_set_operation(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { + let select_depth = depths[select_index]; + let body_end = (select_index + 1..tokens.len()) + .find(|&index| { + depths[index] == select_depth && tokens[index].token_type == TokenType::RParen + }) + .unwrap_or(tokens.len()); + (select_index + 1..body_end).any(|index| { + depths[index] == select_depth + && matches!( + tokens[index].token_type, + TokenType::Union | TokenType::Intersect | TokenType::Except + ) + }) +} + +fn is_top_level_cte_body(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { + let Some(open_index) = significant_before(tokens, select_index) else { + return false; + }; + if tokens[open_index].token_type != TokenType::LParen || depths[open_index] != 0 { + return false; + } + significant_before(tokens, open_index) + .is_some_and(|index| tokens[index].token_type == TokenType::As) +} + +fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQueryFacts { + let query = context.query; + let QuerySlice { + tokens, + direct, + start: from_position, + end: relation_end, + .. + } = query; + let from_position = *from_position; + let relation_end = *relation_end; + let mut facts = AdditionalQueryFacts::default(); + let relations = &direct[from_position + 1..relation_end]; + let relation_count = if is_values_relation(tokens, relations) { + 1 + } else { + 1 + relations + .iter() + .filter(|&&index| { + matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) + }) + .count() + }; + let mut known = relation_names(tokens, relations); + known.extend(context.outer_relations.iter().cloned()); + let references: Vec = direct[..from_position] + .iter() + .chain(direct[relation_end..].iter()) + .copied() + .collect(); + let projection_aliases = projection_alias_positions(tokens, &direct[..from_position]); + let candidate_context = ReferenceCandidateContext { + tokens, + references: &references, + external_identifiers: context.external_identifiers, + projection_aliases: &projection_aliases, + }; + let qualified: Vec = references + .iter() + .enumerate() + .filter_map(|(position, &index)| simple_qualifier(tokens, &references, position, index)) + .collect(); + let unqualified: Vec = references + .iter() + .enumerate() + .filter_map(|(position, &index)| unqualified_reference(&candidate_context, position, index)) + .collect(); + if relation_count > 1 { + facts + .unqualified_multi_source_columns + .extend(unqualified.iter().map(|&index| tokens[index].span)); + } else { + facts + .inconsistent_single_source_qualification + .extend(inconsistent_qualification( + tokens, + (&qualified, &unqualified), + context.sql_macro_outputs, + )); + } + facts.unknown_relation_qualifiers.extend( + qualified + .iter() + .filter(|&&index| !known.contains(&tokens[index].text.to_ascii_lowercase())) + .map(|&index| tokens[index].span), + ); + let projects_bare_star = has_bare_projected_star(tokens, &direct[..from_position]); + if !projects_bare_star { + facts.unused_joined_relations.extend(unused_join_spans( + query, + context.query_start, + context.query_end, + )); + } + facts +} + +/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. In a macro body a plain slot read lints as a pair of macro parameter placeholders and counts as unqualified. In a call's output, a read qualified by a relation the call passes takes neither style: the slot decides how it expands, not the caller or the macro author. A scope whose every reference a macro call generated is judged once, in the macro file. +fn inconsistent_qualification( + tokens: &[Token], + (qualified, unqualified): (&[usize], &[usize]), + outputs: &[SqlMacroOutput], +) -> Vec { + let output_of = |index: usize| -> Option { + let start: usize = tokens[index].span.start; + outputs + .iter() + .position(|output| output.start <= start && start < output.end) + }; + let mut references = qualified.iter().chain(unqualified).copied(); + if let Some(first) = references.next() + && let Some(output) = output_of(first) + && references.all(|index| output_of(index) == Some(output)) + { + return Vec::new(); + } + let passed_relation_read = |index: &usize| { + output_of(*index).is_some_and(|output| { + let qualifier: String = unquoted_name(&tokens[*index].text); + outputs[output] + .relations + .iter() + .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) + }) + }; + let (plain_reads, qualified): (Vec, Vec) = qualified + .iter() + .copied() + .filter(|index| !passed_relation_read(index)) + .partition(|&index| is_plain_slot_read(tokens, index)); + if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { + return Vec::new(); + } + let mut unqualified: Vec = unqualified.to_vec(); + unqualified.extend(plain_reads); + unqualified.sort_unstable(); + unqualified + .iter() + .map(|&index| tokens[index].span) + .collect() +} + +/// An identifier's name without its quotes. +fn unquoted_name(text: &str) -> String { + text.trim_matches(|character| matches!(character, '"' | '`' | '[' | ']')) + .to_owned() +} + +/// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. +fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { + let is_parameter = |token: &Token| { + token + .text + .to_ascii_lowercase() + .starts_with(MACRO_PARAMETER_PLACEHOLDER_PREFIX) + }; + is_parameter(&tokens[index]) && tokens.get(index + 2).is_some_and(is_parameter) +} + +fn simple_qualifier( + tokens: &[Token], + references: &[usize], + position: usize, + index: usize, +) -> Option { + let previous = position.checked_sub(1).map(|value| references[value]); + let dot = references.get(position + 1).copied(); + let column = references.get(position + 2).copied(); + let following = references.get(position + 3).copied(); + (is_identifier(&tokens[index]) + && !is_generated_identifier(&tokens[index]) + && previous.is_none_or(|candidate| { + !matches!( + tokens[candidate].token_type, + TokenType::Dot | TokenType::Colon | TokenType::DColon | TokenType::DotColon + ) + }) + && dot.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot) + && column.is_some_and(|candidate| is_identifier(&tokens[candidate])) + && following.is_none_or(|candidate| tokens[candidate].token_type != TokenType::Dot)) + .then_some(index) +} + +fn unqualified_reference( + context: &ReferenceCandidateContext<'_>, + position: usize, + index: usize, +) -> Option { + let ReferenceCandidateContext { + tokens, + references, + external_identifiers, + projection_aliases, + } = context; + let previous = position.checked_sub(1).map(|value| references[value]); + let next = references.get(position + 1).copied(); + (is_identifier(&tokens[index]) + && !is_generated_identifier(&tokens[index]) + && !is_context_value(&tokens[index]) + && !external_identifiers.contains(&tokens[index].text.to_ascii_lowercase()) + && projection_aliases + .get(&tokens[index].text.to_ascii_lowercase()) + .is_none_or(|alias_index| *alias_index >= index) + && previous.is_none_or(|candidate| { + !matches!( + tokens[candidate].token_type, + TokenType::Dot + | TokenType::As + | TokenType::Colon + | TokenType::DColon + | TokenType::DotColon + ) + }) + && next.is_none_or(|candidate| { + !matches!( + tokens[candidate].token_type, + TokenType::Dot + | TokenType::LParen + | TokenType::Colon + | TokenType::DColon + | TokenType::DotColon + ) + })) + .then_some(index) +} + +fn relation_names(tokens: &[Token], relation_clause: &[usize]) -> FxHashSet { + let mut names: FxHashSet = FxHashSet::default(); + for (position, &index) in relation_clause.iter().enumerate() { + let starts_relation = position == 0 + || matches!( + tokens[relation_clause[position - 1]].token_type, + TokenType::Join | TokenType::Comma + ); + if starts_relation && is_identifier(&tokens[index]) { + names.insert(tokens[index].text.to_ascii_lowercase()); + } + if tokens[index].token_type == TokenType::As + && let Some(&alias) = relation_clause.get(position + 1) + && is_identifier(&tokens[alias]) + { + names.insert(tokens[alias].text.to_ascii_lowercase()); + } + } + let mut segment_start = 0_usize; + for segment_end in relation_clause + .iter() + .enumerate() + .filter_map(|(position, &index)| { + matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) + .then_some(position) + }) + .chain(std::iter::once(relation_clause.len())) + { + let segment = &relation_clause[segment_start..segment_end]; + segment_start = segment_end + 1; + let relation_end = segment + .iter() + .position(|&index| is_relation_condition_start(&tokens[index])) + .unwrap_or(segment.len()); + if let Some(&implicit_name) = segment[..relation_end] + .iter() + .rev() + .find(|&&index| is_identifier(&tokens[index])) + { + names.insert(tokens[implicit_name].text.to_ascii_lowercase()); + } + } + names +} + +fn outer_relation_names( + tokens: &[Token], + depths: &[usize], + select_index: usize, + depth: usize, +) -> FxHashSet { + let mut names: FxHashSet = FxHashSet::default(); + for outer_select in (0..select_index).filter(|&index| { + tokens[index].token_type == TokenType::Select + && depths[index] < depth + && query_end(tokens, depths, index, depths[index]) > select_index + }) { + let outer_depth = depths[outer_select]; + let outer_end = query_end(tokens, depths, outer_select, outer_depth); + let direct: Vec = direct_indices(depths, outer_select + 1, outer_end, outer_depth) + .into_iter() + .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) + .collect(); + let Some(from_position) = direct + .iter() + .position(|&index| is_query_from(tokens, index)) + else { + continue; + }; + let clause_end = direct[from_position + 1..] + .iter() + .position(|&index| is_after_relation_clause(tokens[index].token_type)) + .map_or(direct.len(), |offset| from_position + 1 + offset); + names.extend(relation_names( + tokens, + &direct[from_position + 1..clause_end], + )); + } + names +} + +fn projected_star_spans(tokens: &[Token], projection: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (position, &index) in projection.iter().enumerate() { + if tokens[index].token_type != TokenType::Star { + continue; + } + let previous = position.checked_sub(1).map(|value| projection[value]); + let qualified = + previous.is_some_and(|candidate| tokens[candidate].token_type == TokenType::Dot); + let mut item_start = 0_usize; + for prior_position in (0..position).rev() { + if tokens[projection[prior_position]].token_type == TokenType::Comma { + item_start = prior_position + 1; + break; + } + } + let bare = is_bare_star_prefix(tokens, &projection[item_start..position]); + if qualified || bare { + spans.push(tokens[index].span); + } + } + spans +} + +fn has_bare_projected_star(tokens: &[Token], projection: &[usize]) -> bool { + projection.iter().enumerate().any(|(position, &index)| { + if tokens[index].token_type != TokenType::Star { + return false; + } + let mut item_start = 0_usize; + for prior_position in (0..position).rev() { + if tokens[projection[prior_position]].token_type == TokenType::Comma { + item_start = prior_position + 1; + break; + } + } + is_bare_star_prefix(tokens, &projection[item_start..position]) + }) +} + +fn is_exists_subquery(tokens: &[Token], select_index: usize) -> bool { + let Some(open_index) = significant_before(tokens, select_index) else { + return false; + }; + tokens[open_index].token_type == TokenType::LParen + && significant_before(tokens, open_index) + .is_some_and(|index| tokens[index].token_type == TokenType::Exists) +} + +fn is_set_continuation_select(tokens: &[Token], select_index: usize) -> bool { + let mut previous = significant_before(tokens, select_index); + if previous.is_some_and(|index| { + matches!( + tokens[index].token_type, + TokenType::All | TokenType::Distinct + ) + }) { + previous = previous.and_then(|index| significant_before(tokens, index)); + } + previous.is_some_and(|index| { + matches!( + tokens[index].token_type, + TokenType::Union | TokenType::Intersect | TokenType::Except + ) + }) +} + +fn is_bare_star_prefix(tokens: &[Token], prefix: &[usize]) -> bool { + let mut position = 0_usize; + while prefix.get(position).is_some_and(|&index| { + matches!( + tokens[index].token_type, + TokenType::Distinct | TokenType::All + ) + }) { + position += 1; + } + if position == prefix.len() { + return true; + } + if tokens[prefix[position]].token_type != TokenType::Top { + return false; + } + position += 1; + if !prefix + .get(position) + .is_some_and(|&index| tokens[index].token_type == TokenType::Number) + { + return false; + } + position += 1; + if prefix + .get(position) + .is_some_and(|&index| tokens[index].token_type == TokenType::Percent) + { + position += 1; + } + if prefix + .get(position) + .is_some_and(|&index| tokens[index].token_type == TokenType::With) + && prefix + .get(position + 1) + .is_some_and(|&index| tokens[index].token_type == TokenType::Ties) + { + position += 2; + } + position == prefix.len() +} + +fn is_scalar_subquery(tokens: &[Token], depths: &[usize], select_index: usize) -> bool { + let Some(open_index) = significant_before(tokens, select_index) else { + return false; + }; + if tokens[open_index].token_type != TokenType::LParen { + return false; + } + let Some(previous) = significant_before(tokens, open_index) else { + return false; + }; + if matches!( + tokens[previous].token_type, + TokenType::From | TokenType::Join | TokenType::As | TokenType::Exists | TokenType::Lateral + ) { + return false; + } + if tokens[previous].token_type != TokenType::Comma { + return true; + } + let parent_depth = depths[previous]; + let query_start = (0..previous) + .rev() + .find(|&index| { + tokens[index].token_type == TokenType::Select && depths[index] < parent_depth + }) + .map_or(0, |index| index + 1); + let relation_boundary = (query_start..previous).rev().find(|&index| { + depths[index] == parent_depth + && (tokens[index].token_type == TokenType::Select + || is_query_from(tokens, index) + || matches!( + tokens[index].token_type, + TokenType::Join | TokenType::Semicolon + )) + }); + !relation_boundary.is_some_and(|index| { + is_query_from(tokens, index) || tokens[index].token_type == TokenType::Join + }) +} + +fn is_ceremonial_select(tokens: &[Token], projection: &[usize], allowed: bool) -> bool { + allowed + && projection.len() == 1 + && tokens[projection[0]].token_type == TokenType::Number + && tokens[projection[0]].text == COUNT_ONE_LITERAL +} + +fn is_values_relation(tokens: &[Token], relation_clause: &[usize]) -> bool { + relation_clause + .first() + .is_some_and(|&index| tokens[index].token_type == TokenType::Values) +} + +fn is_generated_identifier(token: &Token) -> bool { + let normalized = token.text.to_ascii_lowercase(); + GENERATED_PLACEHOLDER_PREFIXES + .iter() + .any(|prefix| normalized.starts_with(prefix)) +} + +fn is_relation_condition_start(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::On | TokenType::Using | TokenType::MatchCondition + ) || token.text.eq_ignore_ascii_case("MATCH_CONDITION") +} + +fn is_context_value(token: &Token) -> bool { + matches!( + token.text.to_ascii_uppercase().as_str(), + "CURRENT_CATALOG" + | "CURRENT_DATE" + | "CURRENT_DATETIME" + | "CURRENT_ROLE" + | "CURRENT_SCHEMA" + | "CURRENT_TIME" + | "CURRENT_TIMESTAMP" + | "CURRENT_USER" + ) +} + +fn projection_alias_positions(tokens: &[Token], projection: &[usize]) -> FxHashMap { + let mut aliases: FxHashMap = FxHashMap::default(); + for window in projection.windows(2) { + if tokens[window[0]].token_type == TokenType::As && is_identifier(&tokens[window[1]]) { + aliases.insert(tokens[window[1]].text.to_ascii_lowercase(), window[1]); + } + } + aliases +} + +fn unused_join_spans(query: &QuerySlice<'_>, query_start: usize, query_end: usize) -> Vec { + let QuerySlice { + tokens, + direct, + start: from_position, + end: relation_end, + .. + } = query; + let from_position = *from_position; + let relation_end = *relation_end; + let relation_start_index = direct.get(from_position + 1).copied().unwrap_or(query_end); + let relation_end_index = direct.get(relation_end).copied().unwrap_or(query_end); + let references: Vec = (query_start..relation_start_index) + .chain(relation_end_index..query_end) + .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) + .collect(); + if contains_possible_unqualified_reference(tokens, &references) { + return Vec::new(); + } + let mut spans: Vec = Vec::new(); + for position in from_position + 1..relation_end { + let index = direct[position]; + if tokens[index].token_type != TokenType::Join { + continue; + } + let Some(alias) = relation_alias_after_join(tokens, direct, position) else { + continue; + }; + let used_outside_relation_clause = + contains_qualified_reference(tokens, &references, &alias); + let downstream_relation_start = direct[position + 1..relation_end] + .iter() + .position(|&candidate| { + matches!( + tokens[candidate].token_type, + TokenType::Join | TokenType::Comma + ) + }) + .map(|offset| position + 1 + offset); + let used_by_downstream_relation = downstream_relation_start.is_some_and(|start| { + let downstream_start_index = direct[start]; + let downstream_end_index = direct.get(relation_end).copied().unwrap_or(query_end); + let downstream_references: Vec = (downstream_start_index..downstream_end_index) + .filter(|&candidate| { + !is_layout(&tokens[candidate]) && !is_comment(&tokens[candidate]) + }) + .collect(); + contains_qualified_reference(tokens, &downstream_references, &alias) + }); + let used = used_outside_relation_clause || used_by_downstream_relation; + if !used { + spans.push(tokens[index].span); + } + } + spans +} + +fn contains_possible_unqualified_reference(tokens: &[Token], references: &[usize]) -> bool { + references.iter().enumerate().any(|(position, &index)| { + if !is_identifier(&tokens[index]) + || is_generated_identifier(&tokens[index]) + || is_context_value(&tokens[index]) + { + return false; + } + let previous = if position == 0 { + None + } else { + Some(references[position - 1]) + }; + let next = references.get(position + 1).copied(); + previous.is_none_or(|candidate| { + !matches!(tokens[candidate].token_type, TokenType::Dot | TokenType::As) + }) && next.is_none_or(|candidate| { + !matches!( + tokens[candidate].token_type, + TokenType::Dot | TokenType::LParen + ) + }) + }) +} + +fn contains_qualified_reference(tokens: &[Token], references: &[usize], alias: &str) -> bool { + references.windows(2).any(|window| { + tokens[window[0]].text.eq_ignore_ascii_case(alias) + && tokens[window[1]].token_type == TokenType::Dot + }) +} + +fn is_after_relation_clause(token_type: TokenType) -> bool { + matches!( + token_type, + TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + | TokenType::Fetch + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + | TokenType::Semicolon + ) +} + +fn unaliased_calculation_spans(tokens: &[Token], projection: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + let mut start = 0_usize; + let ends: Vec = projection + .iter() + .enumerate() + .filter_map(|(position, &index)| { + (tokens[index].token_type == TokenType::Comma).then_some(position) + }) + .chain(std::iter::once(projection.len())) + .collect(); + for end in ends { + let item = &projection[start..end]; + start = end + 1; + if item.is_empty() + || item + .iter() + .any(|&index| tokens[index].token_type == TokenType::As) + || item + .iter() + .any(|&index| is_generated_identifier(&tokens[index])) + || simple_projection(tokens, item) + || implicit_alias_projection(tokens, item) + { + continue; + } + spans.push(tokens[item[0]].span); + } + spans +} + +fn simple_projection(tokens: &[Token], item: &[usize]) -> bool { + let trimmed: Vec = item + .iter() + .copied() + .skip_while(|&index| { + matches!( + tokens[index].token_type, + TokenType::Distinct | TokenType::All + ) + }) + .collect(); + let wildcard = trimmed.first().is_some_and(|&index| { + tokens[index].token_type == TokenType::Star + || (trimmed.len() >= QUALIFIED_REFERENCE_LENGTH + && is_identifier(&tokens[index]) + && tokens[trimmed[1]].token_type == TokenType::Dot + && tokens[trimmed[2]].token_type == TokenType::Star) + }); + wildcard + || trimmed.len() == 1 + && (is_bare_column_token(&tokens[trimmed[0]]) + || tokens[trimmed[0]].token_type == TokenType::Star) + || trimmed.len() == QUALIFIED_REFERENCE_LENGTH + && is_identifier(&tokens[trimmed[0]]) + && tokens[trimmed[1]].token_type == TokenType::Dot + && (is_bare_column_token(&tokens[trimmed[2]]) + || tokens[trimmed[2]].token_type == TokenType::Star) +} + +fn is_bare_column_token(token: &Token) -> bool { + is_identifier(token) + || (!matches!( + token.token_type, + TokenType::Null + | TokenType::True + | TokenType::False + | TokenType::Number + | TokenType::String + ) && token + .text + .chars() + .next() + .is_some_and(|character| character.is_alphabetic() || character == '_') + && token + .text + .chars() + .all(|character| character.is_alphanumeric() || character == '_')) +} + +fn implicit_alias_projection(tokens: &[Token], item: &[usize]) -> bool { + item.len() >= IMPLICIT_ALIAS_MINIMUM_LENGTH + && is_identifier(&tokens[*item.last().unwrap_or(&item[0])]) + && !matches!( + tokens[item[item.len() - 2]].token_type, + TokenType::Dot | TokenType::Plus | TokenType::Dash | TokenType::Star | TokenType::Slash + ) +} + +fn unused_alias_spans(query: &QuerySlice<'_>) -> Vec { + let QuerySlice { + tokens, + depths, + direct: relation_clause, + start: query_start, + end: query_end, + depth, + } = query; + let query_start = *query_start; + let query_end = *query_end; + let depth = *depth; + let mut spans: Vec = Vec::new(); + for (span_start, alias_index) in relation_alias_positions(tokens, relation_clause) { + if significant_after(tokens, alias_index) + .is_some_and(|index| tokens[index].token_type == TokenType::LParen) + { + continue; + } + let alias = &tokens[alias_index].text; + let relation_name = significant_before(tokens, span_start); + let used = (query_start..query_end).any(|index| { + index != alias_index + && Some(index) != relation_name + && depths[index] >= depth + && is_identifier(&tokens[index]) + && tokens[index].text.eq_ignore_ascii_case(alias) + && significant_before(tokens, index).is_none_or(|previous| { + !matches!(tokens[previous].token_type, TokenType::Dot | TokenType::As) + }) + && significant_after(tokens, index) + .is_none_or(|next| tokens[next].token_type != TokenType::LParen) + }); + if !used { + spans.push(Span { + start: tokens[span_start].span.start, + end: tokens[alias_index].span.end, + line: tokens[span_start].span.line, + column: tokens[span_start].span.column, + }); + } + } + spans +} + +fn relation_alias_positions(tokens: &[Token], relation_clause: &[usize]) -> Vec<(usize, usize)> { + let mut aliases: Vec<(usize, usize)> = Vec::new(); + let mut segment_start = 0_usize; + let mut in_condition = false; + for (position, &index) in relation_clause.iter().enumerate() { + if matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) { + segment_start = position + 1; + in_condition = false; + continue; + } + if in_condition || is_relation_condition_start(&tokens[index]) { + in_condition = true; + continue; + } + if tokens[index].token_type == TokenType::As { + if let Some(&alias) = relation_clause.get(position + 1) + && is_identifier(&tokens[alias]) + { + aliases.push((index, alias)); + } + continue; + } + if is_implicit_relation_alias( + tokens, + &relation_clause[segment_start..], + position - segment_start, + ) { + aliases.push((index, index)); + } + } + aliases +} + +fn is_implicit_relation_alias(tokens: &[Token], segment: &[usize], position: usize) -> bool { + let candidate = &tokens[segment[position]]; + if position == 0 + || !is_identifier(candidate) + || candidate.token_type != TokenType::QuotedIdentifier + && RELATION_MODIFIER_KEYWORDS + .iter() + .any(|keyword| candidate.text.eq_ignore_ascii_case(keyword)) + { + return false; + } + let previous = &tokens[segment[position - 1]]; + if previous.token_type == TokenType::LParen { + return true; + } + is_identifier(previous) + && position + .checked_sub(QUALIFIER_PAIR_LENGTH) + .is_none_or(|before| tokens[segment[before]].token_type == TokenType::Dot) +} + +fn null_rejected_left_join_spans( + tokens: &[Token], + direct: &[usize], + relation_end: usize, +) -> Vec { + let Some(where_position) = direct + .iter() + .position(|&index| tokens[index].token_type == TokenType::Where) + else { + return Vec::new(); + }; + if direct[where_position + 1..] + .iter() + .any(|&index| tokens[index].token_type == TokenType::Or) + { + return Vec::new(); + } + let mut spans: Vec = Vec::new(); + for position in 0..relation_end { + let index = direct[position]; + if tokens[index].token_type != TokenType::Left { + continue; + } + let Some(join_position) = direct[position + 1..relation_end] + .iter() + .position(|&candidate| tokens[candidate].token_type == TokenType::Join) + .map(|offset| position + 1 + offset) + else { + continue; + }; + let Some(alias) = relation_alias_after_join(tokens, direct, join_position) else { + continue; + }; + if right_side_null_rejected(tokens, &direct[where_position + 1..], &alias) { + spans.push(tokens[index].span); + } + } + spans +} + +fn right_side_null_rejected(tokens: &[Token], predicate: &[usize], alias: &str) -> bool { + for window in predicate.windows(4) { + if !tokens[window[0]].text.eq_ignore_ascii_case(alias) + || tokens[window[1]].token_type != TokenType::Dot + || !is_identifier(&tokens[window[2]]) + { + continue; + } + if matches!( + tokens[window[3]].token_type, + TokenType::Eq + | TokenType::Neq + | TokenType::Lt + | TokenType::Lte + | TokenType::Gt + | TokenType::Gte + | TokenType::Like + | TokenType::In + | TokenType::Between + ) { + return true; + } + } + for window in predicate.windows(6) { + if tokens[window[0]].text.eq_ignore_ascii_case(alias) + && tokens[window[1]].token_type == TokenType::Dot + && is_identifier(&tokens[window[2]]) + && tokens[window[3]].token_type == TokenType::Is + && tokens[window[4]].token_type == TokenType::Not + && tokens[window[5]].token_type == TokenType::Null + { + return true; + } + } + false +} + +fn relation_alias_after_join( + tokens: &[Token], + direct: &[usize], + join_position: usize, +) -> Option { + let relation = &direct[join_position + 1..]; + let relation_end = relation + .iter() + .position(|&index| { + is_relation_condition_start(&tokens[index]) + || is_after_relation_clause(tokens[index].token_type) + || matches!(tokens[index].token_type, TokenType::Join | TokenType::Comma) + }) + .unwrap_or(relation.len()); + relation[..relation_end] + .iter() + .rev() + .find(|&&index| is_identifier(&tokens[index])) + .map(|&index| tokens[index].text.to_ascii_lowercase()) +} + +fn implicit_inner_join_spans(tokens: &[Token], relation_clause: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (position, &index) in relation_clause.iter().enumerate() { + if is_plain_conditioned_join(tokens, relation_clause, position, index) { + spans.push(tokens[index].span); + } + } + spans +} + +fn is_plain_conditioned_join( + tokens: &[Token], + relation_clause: &[usize], + position: usize, + index: usize, +) -> bool { + if tokens[index].token_type != TokenType::Join + || position > 0 + && matches!( + tokens[relation_clause[position - 1]].token_type, + TokenType::Inner + | TokenType::Left + | TokenType::Right + | TokenType::Full + | TokenType::Outer + | TokenType::Cross + | TokenType::Natural + | TokenType::AsOf + | TokenType::Semi + | TokenType::Anti + ) + { + return false; + } + for &candidate in &relation_clause[position + 1..] { + if tokens[candidate].token_type == TokenType::Join { + break; + } + if matches!( + tokens[candidate].token_type, + TokenType::On | TokenType::Using + ) { + return true; + } + } + false +} + +fn ambiguous_order_direction_spans(tokens: &[Token], direct: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (position, &index) in direct.iter().enumerate() { + if tokens[index].token_type != TokenType::Order + || direct + .get(position + 1) + .is_none_or(|&by| tokens[by].token_type != TokenType::By) + { + continue; + } + let end = direct[position + 2..] + .iter() + .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) + .map_or(direct.len(), |offset| position + 2 + offset); + let clause = &direct[position + 2..end]; + let item_ends: Vec = clause + .iter() + .enumerate() + .filter_map(|(item_position, &candidate)| { + (tokens[candidate].token_type == TokenType::Comma).then_some(item_position) + }) + .chain(std::iter::once(clause.len())) + .collect(); + let mut item_start = 0_usize; + let mut explicit = false; + let mut implicit = false; + for item_end in item_ends { + let item = &clause[item_start..item_end]; + item_start = item_end + 1; + let has_direction = item.last().is_some_and(|&candidate| { + matches!( + tokens[candidate].token_type, + TokenType::Asc | TokenType::Desc + ) + }); + explicit |= has_direction; + implicit |= !has_direction && !item.is_empty(); + } + if explicit && implicit { + spans.push(tokens[index].span); + } + } + spans +} + +fn duplicate_alias_spans(tokens: &[Token], indices: &[usize]) -> Vec { + let mut seen: FxHashSet = FxHashSet::default(); + let mut duplicates: Vec = Vec::new(); + for window in indices.windows(2) { + if tokens[window[0]].token_type != TokenType::As || !is_identifier(&tokens[window[1]]) { + continue; + } + let normalized = tokens[window[1]].text.to_ascii_lowercase(); + if !seen.insert(normalized) { + duplicates.push(tokens[window[1]].span); + } + } + duplicates +} + +fn mixed_reference_clause_spans(tokens: &[Token], direct: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (position, &index) in direct.iter().enumerate() { + if !matches!( + tokens[index].token_type, + TokenType::Group | TokenType::Order + ) || direct + .get(position + 1) + .is_none_or(|&by| tokens[by].token_type != TokenType::By) + { + continue; + } + let end = direct[position + 2..] + .iter() + .position(|&candidate| is_clause_boundary(tokens[candidate].token_type)) + .map_or(direct.len(), |offset| position + 2 + offset); + let clause = &direct[position + 2..end]; + let mut has_number = false; + let mut has_name = false; + for (item_position, &candidate) in clause.iter().enumerate() { + if item_position == 0 + || tokens[clause[item_position - 1]].token_type == TokenType::Comma + { + has_number |= tokens[candidate].token_type == TokenType::Number; + has_name |= tokens[candidate].token_type != TokenType::Number; + } + } + if has_number && has_name { + spans.push(tokens[index].span); + } + } + spans +} + +fn is_clause_boundary(token_type: TokenType) -> bool { + matches!( + token_type, + TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + | TokenType::Fetch + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + | TokenType::Semicolon + ) +} + +fn parenthesized_distinct_span( + tokens: &[Token], + depths: &[usize], + open_index: usize, +) -> Option { + let close_depth = depths[open_index] + 1; + let close_index = (open_index + 1..tokens.len()).find(|&index| { + tokens[index].token_type == TokenType::RParen && depths[index] == close_depth + })?; + Some(Span { + start: tokens[significant_before(tokens, open_index)?].span.start, + end: tokens[close_index].span.end, + line: tokens[significant_before(tokens, open_index)?].span.line, + column: tokens[significant_before(tokens, open_index)?].span.column, + }) +} + +fn count_one_span(tokens: &[Token], significant: &[usize], position: usize) -> Option { + let open = *significant.get(position + 1)?; + let one = *significant.get(position + 2)?; + let close = *significant.get(position + 3)?; + (tokens[open].token_type == TokenType::LParen + && tokens[one].token_type == TokenType::Number + && tokens[one].text == COUNT_ONE_LITERAL + && tokens[close].token_type == TokenType::RParen) + .then_some(tokens[one].span) +} + +fn simple_boolean_case_span( + tokens: &[Token], + significant: &[usize], + position: usize, +) -> Option { + let tail = &significant[position + 1..]; + let end_offset = tail + .iter() + .position(|&index| tokens[index].token_type == TokenType::End)?; + let body = &tail[..=end_offset]; + if body + .iter() + .any(|&index| tokens[index].token_type == TokenType::Case) + { + return None; + } + let when_position = body + .iter() + .position(|&index| tokens[index].token_type == TokenType::When)?; + let else_position = body + .iter() + .position(|&index| tokens[index].token_type == TokenType::Else)?; + let else_value = *body.get(else_position + 1)?; + let mut thens = body + .iter() + .enumerate() + .filter(|&(_, &index)| tokens[index].token_type == TokenType::Then) + .peekable(); + thens.peek()?; + let every_branch_returns_true = thens.all(|(offset, _)| { + body.get(offset + 1) + .is_some_and(|&value| tokens[value].token_type == TokenType::True) + && body.get(offset + 2).is_some_and(|&next| { + matches!(tokens[next].token_type, TokenType::When | TokenType::Else) + }) + }); + if when_position != 0 + || !every_branch_returns_true + || tokens[else_value].token_type != TokenType::False + || else_position + 2 != body.len() - 1 + { + return None; + } + let start = tokens[significant[position]].span; + let end = tokens[*body.last()?].span; + Some(Span { + start: start.start, + end: end.end, + line: start.line, + column: start.column, + }) +} + +fn unstable_row_number_span( + tokens: &[Token], + depths: &[usize], + function_index: usize, +) -> Option { + let over_index = (function_index + 1..tokens.len()).find(|&index| { + depths[index] == depths[function_index] && tokens[index].token_type == TokenType::Over + })?; + let open_index = significant_after(tokens, over_index)?; + if tokens[open_index].token_type != TokenType::LParen { + return None; + } + let close_depth = depths[open_index] + 1; + let close_index = (open_index + 1..tokens.len()).find(|&index| { + tokens[index].token_type == TokenType::RParen && depths[index] == close_depth + })?; + (!tokens[open_index + 1..close_index] + .iter() + .any(|token| matches!(token.token_type, TokenType::Order | TokenType::OrderBy))) + .then_some(tokens[function_index].span) +} + +fn null_not_in_span( + tokens: &[Token], + depths: &[usize], + significant: &[usize], + position: usize, +) -> Option { + let in_index = *significant.get(position + 1)?; + let open_index = *significant.get(position + 2)?; + if tokens[in_index].token_type != TokenType::In + || tokens[open_index].token_type != TokenType::LParen + { + return None; + } + let close_depth = depths[open_index] + 1; + let close_index = (open_index + 1..tokens.len()).find(|&index| { + tokens[index].token_type == TokenType::RParen && depths[index] == close_depth + })?; + let list_depth = depths[open_index] + 1; + let list = &tokens[open_index + 1..close_index]; + (!list + .iter() + .any(|token| token.token_type == TokenType::Select) + && list.iter().enumerate().any(|(offset, token)| { + token.token_type == TokenType::Null && depths[open_index + 1 + offset] == list_depth + })) + .then_some(tokens[significant[position]].span) +} + +fn constant_predicate(tokens: &[Token], indices: &[usize]) -> bool { + let left = &tokens[indices[0]]; + let operator = &tokens[indices[1]]; + let right = &tokens[indices[2]]; + matches!( + left.token_type, + TokenType::Number | TokenType::String | TokenType::True | TokenType::False + ) && left.token_type == right.token_type + && left.text == right.text + && matches!(operator.token_type, TokenType::Eq | TokenType::Neq) +} + +fn set_arity_mismatch_spans(tokens: &[Token], depths: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if !matches!( + token.token_type, + TokenType::Union | TokenType::Intersect | TokenType::Except + ) { + continue; + } + let depth = depths[index]; + let Some(left_select) = (0..index).rev().find(|&candidate| { + depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select + }) else { + continue; + }; + let Some(right_select) = (index + 1..tokens.len()).find(|&candidate| { + depths[candidate] == depth && tokens[candidate].token_type == TokenType::Select + }) else { + continue; + }; + let left_query = QuerySlice { + tokens, + depths, + direct: &[], + start: left_select, + end: index, + depth, + }; + let left_count = projection_arity(&left_query); + let right_end = query_end(tokens, depths, right_select, depth); + let right_query = QuerySlice { + tokens, + depths, + direct: &[], + start: right_select, + end: right_end, + depth, + }; + let right_count = projection_arity(&right_query); + if left_count + .zip(right_count) + .is_some_and(|(left, right)| left != right) + { + spans.push(token.span); + } + } + spans +} + +fn projection_arity(query: &QuerySlice<'_>) -> Option { + let QuerySlice { + tokens, + depths, + start: select, + end, + depth, + .. + } = query; + let select = *select; + let end = *end; + let depth = *depth; + let projection_end = (select + 1..end) + .find(|&index| depths[index] == depth && is_query_from(tokens, index)) + .unwrap_or(end); + if (select + 1..projection_end) + .any(|index| depths[index] == depth && tokens[index].token_type == TokenType::Star) + { + return None; + } + Some( + 1 + (select + 1..projection_end) + .filter(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Comma) + .count(), + ) +} + +fn is_identifier(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var + ) +} diff --git a/crates/sqlquack-lint/src/lint/_helpers/engine.rs b/crates/sqlquack-lint/src/lint/_helpers/engine.rs new file mode 100644 index 000000000..febc368aa --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/engine.rs @@ -0,0 +1,1959 @@ +use std::borrow::Cow; +use std::collections::{HashMap, HashSet, VecDeque}; + +use sqlquack_syntax::expressions::queries::models::With; +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; +use sqlquack_syntax::{Dialect, Expression, ExpressionWalk}; + +use crate::lint::constants::{LINT_API_VERSION, SELECT_ITEM_END_TOKENS}; +use crate::lint::models::{ + AdditionalFactOptions, CrossJoinRequest, FixtureHarness, LintDiagnostic, LintEdit, LintRequest, + LintResponse, LintRuleMetadata, ParsedSql, QueryFacts, SqlMacroOutput, +}; + +use crate::lint::_helpers::additional::collect_additional_facts; +use crate::lint::_helpers::inline_relations::collect_inline_query_relation_spans; +use crate::lint::_helpers::join_predicates::{ + condition_tokens, cross_join_diagnostics, trivial_on_condition, +}; +use crate::lint::_helpers::parsing::{dialect as parse_dialect, parse}; +use crate::lint::_helpers::terminal_shape::{ + collect_terminal_shape_facts, first_row_selection_clause, limits_to_no_rows, + projects_only_constants, +}; + +const NULL_COMPARISON: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN001", + message: "Comparison with NULL is never true", + remediation: "Use IS NULL or IS NOT NULL when testing for NULL.", +}; + +const IMPLICIT_CARTESIAN_JOIN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN002", + message: "Comma-separated sources create an implicit cartesian join", + remediation: "Replace comma-separated sources with an explicit keyed join, or use CROSS JOIN when the cartesian product is intentional.", +}; +const JOIN_WITHOUT_CONDITION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN003", + message: "Non-cross join has no meaningful condition", + remediation: "Add a meaningful ON or USING condition, or declare an intentional cartesian product with CROSS JOIN.", +}; +const UNORDERED_LIMIT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN004", + message: "Row selection is nondeterministic", + remediation: "Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET.", +}; +const UNUSED_CTE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN005", + message: "CTE is unreachable from the final query", + remediation: "Reference the CTE from the final query or another reachable CTE, or remove it.", +}; +const REDUNDANT_DISTINCT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN006", + message: "DISTINCT is redundant with the grouped output", + remediation: "Remove DISTINCT; the equivalent GROUP BY already determines the output groups.", +}; +const POSITIONAL_SET_STAR: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN007", + message: "Positional set operation is vulnerable to column-order drift", + remediation: "Enumerate columns in the same order in every set-operation branch.", +}; +const EXPLICIT_UNION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN008", + message: "Bare UNION leaves duplicate handling implicit", + remediation: "Use UNION DISTINCT or UNION ALL to state duplicate handling explicitly.", +}; +const DUPLICATE_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN009", + message: "Table alias is duplicated in the same query scope", + remediation: "Give every relation in this query scope a unique alias.", +}; +const DUPLICATE_OUTPUT_ALIAS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN010", + message: "Output alias is duplicated in the same select list", + remediation: "Give every projected expression a unique output alias.", +}; +const MIXED_GROUP_ORDER_REFERENCES: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN011", + message: "Clause mixes positional and named references", + remediation: "Use column names or positions consistently throughout this clause.", +}; +const REDUNDANT_ELSE_NULL: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN012", + message: "ELSE NULL is redundant", + remediation: "Remove ELSE NULL; CASE expressions return NULL when ELSE is omitted.", +}; +const PARENTHESIZED_DISTINCT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN013", + message: "DISTINCT is written as though it were a function", + remediation: "Write DISTINCT as a SELECT modifier without function-like parentheses.", +}; +const CONSTANT_PREDICATE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN014", + message: "Predicate compares identical constant values", + remediation: "Remove the scaffold predicate or replace it with the intended condition.", +}; +const CONSECUTIVE_SEMICOLON: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN015", + message: "Empty statement follows another terminator", + remediation: "Remove the redundant statement terminator.", +}; +const REDUNDANT_SELF_ALIAS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN016", + message: "Expression is aliased to its existing name", + remediation: "Remove the self-alias when the inferred output identity remains stable.", +}; +const COUNT_ONE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN017", + message: "Row count uses a constant expression", + remediation: "Use COUNT(*) to state that rows, rather than a nullable expression, are counted.", +}; +const UNSTABLE_ROW_NUMBER: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN018", + message: "ROW_NUMBER has no deterministic ordering", + remediation: "Add ORDER BY with a stable tie-breaker inside the window specification.", +}; +const NULL_NOT_IN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN019", + message: "NOT IN list contains NULL and can reject every row", + remediation: "Remove NULL from the list or express the anti-join with a NULL-safe NOT EXISTS.", +}; +const SET_ARITY_MISMATCH: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN020", + message: "Set-operation branches project different column counts", + remediation: "Project the same number of columns in every set-operation branch.", +}; +const PROJECTED_STAR: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN021", + message: "Query output has an uncontrolled column shape", + remediation: "Enumerate the intended output columns explicitly.", +}; +const UNALIASED_CALCULATION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN022", + message: "Calculated projection has no stable output name", + remediation: "Add an explicit AS alias that describes the projected value.", +}; +const UNUSED_TABLE_ALIAS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN023", + message: "Table alias is never referenced", + remediation: "Use the alias to qualify references or remove it.", +}; +const NULL_REJECTED_LEFT_JOIN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN024", + message: "WHERE predicate rejects NULL rows from a LEFT JOIN", + remediation: "Move the right-side predicate into ON or use INNER JOIN when row rejection is intentional.", +}; +const IMPLICIT_INNER_JOIN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN025", + message: "JOIN type is implicit", + remediation: "Write INNER JOIN explicitly.", +}; +const AMBIGUOUS_ORDER_DIRECTIONS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN026", + message: "ORDER BY mixes explicit and implicit directions", + remediation: "State ASC or DESC for every ordering expression in this clause.", +}; +const UNQUALIFIED_MULTI_SOURCE_COLUMN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN027", + message: "Column reference is unqualified in a multi-relation query", + remediation: "Qualify the column with its relation alias.", +}; +const INCONSISTENT_SINGLE_SOURCE_QUALIFICATION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN028", + message: "Single-relation query mixes qualified and unqualified columns", + remediation: "Use one consistent qualification style in this query scope.", +}; +const UNKNOWN_RELATION_QUALIFIER: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN029", + message: "Qualified reference uses a relation absent from this query scope", + remediation: "Use an available relation alias or add the intended relation to FROM.", +}; +const SIMPLE_BOOLEAN_CASE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN030", + message: "CASE expression only converts a condition to TRUE or FALSE", + remediation: "Use a NULL-safe boolean expression directly.", +}; +const NESTED_ELSE_CASE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN031", + message: "CASE expression is nested directly inside ELSE", + remediation: "Flatten the nested branch into the surrounding CASE expression.", +}; +const UNUSED_JOINED_RELATION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN032", + message: "Joined relation contributes no referenced values", + remediation: "Use the joined relation or remove the join. When the join intentionally filters rows, record it as a reviewed rule exception with a reason in the configuration file.", +}; +const COMMENT_ATTACHMENT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN033", + message: "Comment is not attached to the syntax node it explains", + remediation: "Use a standalone -- comment block immediately above the relevant CTE, JOIN, filter, predicate, selected expression, CASE branch, grouping, or ordering node.", +}; +const CTE_ONLY_BODY: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN034", + message: "Query keeps transformation logic outside top-level CTEs", + remediation: "Move transformation logic into named top-level CTEs before the terminal SELECT.", +}; +const TERMINAL_SELECT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN035", + message: "Terminal SELECT does not read plainly from the final top-level CTE", + remediation: "Name the final logical CTE and leave one terminal SELECT from that CTE.", +}; +const NESTED_CTE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN036", + message: "CTE is nested inside another query", + remediation: "Hoist the nested CTE into the statement's top-level WITH list.", +}; +const RECURSIVE_CTE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN037", + message: "Recursive CTE is not permitted", + remediation: "Replace recursion with an explicit upstream relation or a bounded non-recursive query shape.", +}; +const CROSS_JOIN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN038", + message: "CROSS JOIN creates an explicit cartesian product", + remediation: "Use a keyed JOIN, or make one side provably at most one row (LIMIT 1, an aggregate without GROUP BY, or a one-row VALUES). When the cartesian product is intentional, record it as a reviewed [[rules.rule_exceptions]] entry with rule, path, relation and reason in the configuration file.", +}; +const INLINE_QUERY_RELATION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN039", + message: "Inline query relation obscures data flow", + remediation: "Extract the derived relation into a named top-level CTE; use explicit LATERAL only when the relation must remain correlated.", +}; +const JOIN_EXPRESSION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN040", + message: "JOIN ON contains a computed expression", + remediation: "Compute keys, including variant path access, in input CTEs and join on plain columns. Literal-only predicates such as ON TRUE are allowed.", +}; +const FINAL_CTE_NAME: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN041", + message: "Only the last top-level CTE may be named final, and it must use that name", + remediation: "Name the last top-level CTE final and rename any other final CTE; SQLCLEAN035 checks the terminal SELECT.", +}; + +pub(crate) const DEFAULT_RULES: [&str; 19] = [ + NULL_COMPARISON.code, + IMPLICIT_CARTESIAN_JOIN.code, + JOIN_WITHOUT_CONDITION.code, + UNORDERED_LIMIT.code, + UNUSED_CTE.code, + REDUNDANT_DISTINCT.code, + POSITIONAL_SET_STAR.code, + DUPLICATE_TABLE_ALIAS.code, + DUPLICATE_OUTPUT_ALIAS.code, + UNSTABLE_ROW_NUMBER.code, + NULL_NOT_IN.code, + SET_ARITY_MISMATCH.code, + INLINE_QUERY_RELATION.code, + crate::quality::constants::DUPLICATE_CTE_NAME.code, + crate::quality::constants::NULL_MATCH.code, + crate::quality::constants::LITERAL_ZERO_DIVISION.code, + crate::quality::constants::NATURAL_JOIN.code, + crate::quality::constants::REVERSED_BETWEEN.code, + crate::quality::constants::MISPLACED_AGGREGATE.code, +]; + +const ALL_RULE_METADATA: [&LintRuleMetadata; 53] = [ + &NULL_COMPARISON, + &IMPLICIT_CARTESIAN_JOIN, + &JOIN_WITHOUT_CONDITION, + &UNORDERED_LIMIT, + &UNUSED_CTE, + &REDUNDANT_DISTINCT, + &POSITIONAL_SET_STAR, + &EXPLICIT_UNION, + &DUPLICATE_TABLE_ALIAS, + &DUPLICATE_OUTPUT_ALIAS, + &MIXED_GROUP_ORDER_REFERENCES, + &REDUNDANT_ELSE_NULL, + &PARENTHESIZED_DISTINCT, + &CONSTANT_PREDICATE, + &CONSECUTIVE_SEMICOLON, + &REDUNDANT_SELF_ALIAS, + &COUNT_ONE, + &UNSTABLE_ROW_NUMBER, + &NULL_NOT_IN, + &SET_ARITY_MISMATCH, + &PROJECTED_STAR, + &UNALIASED_CALCULATION, + &UNUSED_TABLE_ALIAS, + &NULL_REJECTED_LEFT_JOIN, + &IMPLICIT_INNER_JOIN, + &AMBIGUOUS_ORDER_DIRECTIONS, + &UNQUALIFIED_MULTI_SOURCE_COLUMN, + &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, + &UNKNOWN_RELATION_QUALIFIER, + &SIMPLE_BOOLEAN_CASE, + &NESTED_ELSE_CASE, + &UNUSED_JOINED_RELATION, + &COMMENT_ATTACHMENT, + &CTE_ONLY_BODY, + &TERMINAL_SELECT, + &NESTED_CTE, + &RECURSIVE_CTE, + &CROSS_JOIN, + &INLINE_QUERY_RELATION, + &JOIN_EXPRESSION, + &FINAL_CTE_NAME, + &crate::quality::constants::UNUSED_CTE_OUTPUT, + &crate::quality::constants::RANKING_SORT_CAP, + &crate::quality::constants::LONG_LITERAL, + &crate::quality::constants::REPEATED_JSON_PARSE, + &crate::quality::constants::DUPLICATE_CTE_NAME, + &crate::quality::constants::NULL_MATCH, + &crate::quality::constants::LITERAL_ZERO_DIVISION, + &crate::quality::constants::UNFILTERED_WRITE, + &crate::quality::constants::INSERT_WITHOUT_COLUMNS, + &crate::quality::constants::NATURAL_JOIN, + &crate::quality::constants::REVERSED_BETWEEN, + &crate::quality::constants::MISPLACED_AGGREGATE, +]; + +/// Whether the harness generated the CTE `name`. +fn is_harness_cte(harness: &FixtureHarness, name: &str) -> bool { + harness + .cte_prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_str())) +} + +struct QueryTokenContext<'a> { + tokens: &'a [Token], + depths: &'a [usize], + direct: &'a [usize], + query_end: usize, + depth: usize, +} + +struct ExpressionRange { + start: usize, + end: usize, + depth: usize, + strip_alias: bool, +} + +struct DiagnosticContext<'a> { + sql: &'a str, + dialect: Dialect, + facts: &'a QueryFacts, + tokens: &'a [Token], + enabled: &'a HashSet, +} + +struct FactBuildOptions<'a> { + external_identifiers: &'a HashSet, + sql_macro_outputs: &'a [SqlMacroOutput], + dependency_identifiers: &'a HashSet, + externally_referenced_ctes: &'a HashSet, + allows_ceremonial_select: bool, + allows_dynamic_output_star: bool, + fixture_harness: Option<&'a FixtureHarness>, +} + +pub(crate) fn lint_json_impl(request_json: &str) -> Result { + let request: LintRequest = + serde_json::from_str(request_json).map_err(|error| error.to_string())?; + let response = lint(request)?; + serde_json::to_string(&response).map_err(|error| error.to_string()) +} + +pub(crate) fn rule_metadata() -> Vec { + ALL_RULE_METADATA.iter().map(|rule| **rule).collect() +} + +pub(crate) fn lint(request: LintRequest) -> Result { + if request.version != LINT_API_VERSION { + return Err(format!( + "unsupported native lint request version {}; expected {LINT_API_VERSION}", + request.version + )); + } + let enabled = enabled_rules(request.enabled_rules, &request.ignored_rules)?; + let Some(dialect) = parse_dialect(&request.dialect)? else { + return Ok(LintResponse { + version: LINT_API_VERSION, + diagnostics: Vec::new(), + unavailable: Some(format!( + "SQL lint is unavailable for the {} dialect: SQLQuack does not parse it yet", + request.dialect + )), + }); + }; + let ParsedSql { + dialect: dialect_type, + tokens, + statements, + } = parse( + &request.sql, + dialect, + request + .fixture_harness + .as_ref() + .and_then(|harness| harness.empty_relation.as_deref()), + )?; + let external_identifiers: HashSet = request + .external_identifiers + .iter() + .map(|identifier| identifier.to_ascii_lowercase()) + .collect(); + let dependency_identifiers: HashSet = request + .dependency_identifiers + .iter() + .map(|identifier| identifier.to_ascii_lowercase()) + .collect(); + let externally_referenced_ctes: HashSet = request + .externally_referenced_ctes + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect(); + let options = FactBuildOptions { + external_identifiers: &external_identifiers, + sql_macro_outputs: &request.sql_macro_outputs, + dependency_identifiers: &dependency_identifiers, + externally_referenced_ctes: &externally_referenced_ctes, + allows_ceremonial_select: request.fixture_harness.is_some(), + allows_dynamic_output_star: request.allows_dynamic_output_star, + fixture_harness: request.fixture_harness.as_ref(), + }; + let facts = build_facts(&statements, &tokens, &request.sql, &options); + let context = DiagnosticContext { + sql: &request.sql, + dialect: dialect_type, + facts: &facts, + tokens: &tokens, + enabled: &enabled, + }; + let mut diagnostics = diagnostics(&context); + if enabled.contains(CROSS_JOIN.code) { + let cross_joins = CrossJoinRequest { + statements: &statements, + tokens: &tokens, + cross_spans: &facts.cross_joins, + relation_keys: &request.relation_keys, + relation_names: &request.relation_names, + }; + diagnostics.extend(cross_join_diagnostics(&cross_joins, &CROSS_JOIN)); + } + let depths = token_depths(&tokens); + if enabled.contains(JOIN_EXPRESSION.code) { + diagnostics.extend(diagnostics_for_spans( + &JOIN_EXPRESSION, + &crate::lint::_helpers::join_predicates::join_expression_spans( + &tokens, + &depths, + dialect_type, + ), + Some("computing a join key requires an authored CTE rewrite"), + )); + } + if enabled.contains(FINAL_CTE_NAME.code) && request.fixture_harness.is_none() { + diagnostics.extend(diagnostics_for_spans( + &FINAL_CTE_NAME, + &crate::lint::_helpers::terminal_shape::final_cte_name_spans(&tokens, &depths), + Some("renaming a CTE requires updating its references"), + )); + } + diagnostics.extend(crate::quality::main::diagnostics::diagnostics( + &crate::quality::models::QualityRequest { + statements: &statements, + tokens: &tokens, + dialect: dialect_type, + enabled: &enabled, + max_literal_length: request + .max_literal_length + .unwrap_or(crate::quality::constants::DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by: request + .max_ranking_order_by + .unwrap_or(crate::quality::constants::DEFAULT_MAX_RANKING_ORDER_BY), + relation_keys: &request.relation_keys, + externally_referenced_ctes: &externally_referenced_ctes, + fixture_body: request.fixture_harness.is_some(), + config_file: request + .config_file + .as_deref() + .unwrap_or(crate::constants::DEFAULT_CONFIG_FILE), + long_literal_guidance: request + .long_literal_guidance + .as_deref() + .unwrap_or(crate::quality::constants::LONG_LITERAL_GUIDANCE), + }, + )?); + diagnostics.sort_by_key(|diagnostic| (diagnostic.start, diagnostic.end, diagnostic.code)); + Ok(LintResponse { + version: LINT_API_VERSION, + diagnostics, + unavailable: None, + }) +} + +fn enabled_rules( + requested: Option>, + ignored: &[String], +) -> Result, String> { + let selectors = + requested.unwrap_or_else(|| DEFAULT_RULES.iter().map(ToString::to_string).collect()); + let mut rules = expand_rule_selectors(&selectors)?; + for code in expand_rule_selectors(ignored)? { + rules.remove(&code); + } + Ok(rules) +} + +fn expand_rule_selectors(selectors: &[String]) -> Result, String> { + let mut selected: HashSet = HashSet::new(); + for selector in selectors { + let matches: Vec<&str> = ALL_RULE_METADATA + .iter() + .map(|rule| rule.code) + .filter(|code| code.starts_with(selector)) + .collect(); + if matches.is_empty() { + return Err(format!("unknown native lint rule '{selector}'")); + } + selected.extend(matches.into_iter().map(ToString::to_string)); + } + Ok(selected) +} + +fn build_facts( + expressions: &[Expression], + tokens: &[Token], + sql: &str, + options: &FactBuildOptions<'_>, +) -> QueryFacts { + let mut facts: QueryFacts = + collect_token_query_facts(tokens, sql, options.allows_ceremonial_select); + facts.null_comparisons = null_comparison_spans(tokens); + facts.additional = collect_additional_facts( + tokens, + &AdditionalFactOptions { + external_identifiers: options.external_identifiers, + sql_macro_outputs: options.sql_macro_outputs, + dependency_identifiers: options.dependency_identifiers, + allows_ceremonial_select: options.allows_ceremonial_select, + allows_dynamic_output_star: options.allows_dynamic_output_star, + fixture_harness: options.fixture_harness, + }, + ); + for expression in expressions { + facts + .unused_cte_names + .extend(collect_unused_ctes(expression, options)); + } + facts +} + +fn collect_token_query_facts( + tokens: &[Token], + sql: &str, + allows_ceremonial_select: bool, +) -> QueryFacts { + let depths = token_depths(tokens); + let mut facts = collect_rule_migration_facts(tokens, &depths, sql, allows_ceremonial_select); + for (select_index, token) in tokens.iter().enumerate() { + if token.token_type != TokenType::Select { + continue; + } + let depth = depths[select_index]; + let end = query_end(tokens, &depths, select_index, depth); + let direct = direct_indices(&depths, select_index + 1, end, depth); + let order = first_type(tokens, &direct, TokenType::Order); + let limit = first_row_selection_clause(tokens, &direct); + if order.is_none() + && let Some(index) = limit + && !limits_to_no_rows(tokens, index) + && !projects_only_constants(tokens, &direct, index) + { + facts.unordered_limits.push(tokens[index].span); + } + let context = QueryTokenContext { + tokens, + depths: &depths, + direct: &direct, + query_end: end, + depth, + }; + if let Some(span) = redundant_distinct_span(&context) { + facts.redundant_distincts.push(span); + } + let (implicit, missing_conditions) = collect_from_and_join_facts(tokens, &direct); + facts.implicit_cartesian_joins.extend(implicit); + facts.joins_without_condition.extend(missing_conditions); + } + facts.positional_set_stars = collect_set_operation_facts(tokens, &depths); + facts +} + +fn collect_rule_migration_facts( + tokens: &[Token], + depths: &[usize], + sql: &str, + allows_ceremonial_select: bool, +) -> QueryFacts { + let mut facts = QueryFacts::default(); + let significant: Vec = (0..tokens.len()) + .filter(|&index| !is_layout(&tokens[index]) && !is_comment(&tokens[index])) + .collect(); + + let root_with = crate::lint::_helpers::terminal_shape::root_with_indices(tokens, depths); + + for (position, &index) in significant.iter().enumerate() { + let text = tokens[index].text.as_str(); + if text.eq_ignore_ascii_case("cross") + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].token_type == TokenType::Join) + { + facts.cross_joins.push(tokens[index].span); + } + if tokens[index].token_type == TokenType::With { + let starts_hierarchy = position.checked_sub(1).is_some_and(|previous| { + tokens[significant[previous]].token_type == TokenType::Start + }); + if !root_with.contains(&index) && !starts_hierarchy { + facts.nested_ctes.push(tokens[index].span); + } + if !starts_hierarchy + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].text.eq_ignore_ascii_case("recursive")) + { + facts.recursive_ctes.push(tokens[index].span); + } + } + } + facts + .inline_query_relations + .extend(collect_inline_query_relation_spans( + tokens, + depths, + &significant, + )); + + let (cte_only_bodies, terminal_selects) = + collect_terminal_shape_facts(tokens, depths, &significant, allows_ceremonial_select); + facts.cte_only_bodies.extend(cte_only_bodies); + facts.terminal_selects.extend(terminal_selects); + facts + .invalid_comment_attachments + .extend(collect_comment_attachment_facts(tokens, sql)); + facts +} + +fn collect_comment_attachment_facts(tokens: &[Token], sql: &str) -> Vec { + let mut invalid: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if !token.trailing_comments.is_empty() { + invalid.push(token.span); + } + if token.comments.is_empty() { + continue; + } + let gap_start = index + .checked_sub(1) + .map_or(0, |previous| tokens[previous].span.end); + let gap = char_slice(sql, gap_start, token.span.start).unwrap_or_default(); + let block_comment = gap.contains("/*"); + let detached = gap.replace("\r\n", "\n").ends_with("\n\n"); + let unsupported_target = matches!( + token.token_type, + TokenType::Comma + | TokenType::RParen + | TokenType::Semicolon + | TokenType::Then + | TokenType::Else + | TokenType::From + ); + if block_comment || detached || unsupported_target { + invalid.push(token.span); + } + } + invalid +} + +pub(super) fn token_depths(tokens: &[Token]) -> Vec { + let mut depth = 0_usize; + tokens + .iter() + .map(|token| { + let current = depth; + if matches!( + token.token_type, + TokenType::LParen | TokenType::LBracket | TokenType::LBrace + ) { + depth += 1; + } else if matches!( + token.token_type, + TokenType::RParen | TokenType::RBracket | TokenType::RBrace + ) { + depth = depth.saturating_sub(1); + } + current + }) + .collect() +} + +pub(super) fn query_end(tokens: &[Token], depths: &[usize], start: usize, depth: usize) -> usize { + (start + 1..tokens.len()) + .find(|&index| { + depths[index] < depth + || depths[index] == depth + && matches!( + tokens[index].token_type, + TokenType::RParen + | TokenType::Semicolon + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + ) + }) + .unwrap_or(tokens.len()) +} + +pub(super) fn direct_indices( + depths: &[usize], + start: usize, + end: usize, + depth: usize, +) -> Vec { + (start..end) + .filter(|&index| depths[index] == depth) + .collect() +} + +fn first_type(tokens: &[Token], indices: &[usize], token_type: TokenType) -> Option { + indices + .iter() + .copied() + .find(|&index| tokens[index].token_type == token_type) +} + +pub(super) fn is_query_from(tokens: &[Token], index: usize) -> bool { + tokens[index].token_type == TokenType::From + && significant_before(tokens, index) + .is_none_or(|previous| tokens[previous].token_type != TokenType::Distinct) +} + +fn redundant_distinct_span(context: &QueryTokenContext<'_>) -> Option { + let QueryTokenContext { + tokens, + depths, + direct, + query_end, + depth, + } = context; + let group_position = group_by_position(tokens, direct)?; + let group_index = direct[group_position]; + let projection_end = direct + .iter() + .copied() + .find(|&index| is_query_from(tokens, index)) + .filter(|index| *index < group_index) + .unwrap_or(group_index); + let distinct_position = direct.iter().position(|&index| { + index < projection_end && tokens[index].token_type == TokenType::Distinct + })?; + if direct + .get(distinct_position + 1) + .is_some_and(|&index| tokens[index].token_type == TokenType::On) + { + return None; + } + let projection_start = direct[distinct_position] + 1; + let group_start = *direct.get(group_position + 2)?; + let group_end = direct[group_position + 2..] + .iter() + .copied() + .find(|&index| { + matches!( + tokens[index].token_type, + TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + ) + }) + .unwrap_or(*query_end); + let projection_signatures = expression_signatures( + tokens, + depths, + &ExpressionRange { + start: projection_start, + end: projection_end, + depth: *depth, + strip_alias: true, + }, + ); + let group_signatures = expression_signatures( + tokens, + depths, + &ExpressionRange { + start: group_start, + end: group_end, + depth: *depth, + strip_alias: false, + }, + ); + if group_signatures.is_empty() + || !group_signatures + .iter() + .all(|group| projection_signatures.contains(group)) + { + return None; + } + Some(tokens[direct[distinct_position]].span) +} + +fn group_by_position(tokens: &[Token], direct: &[usize]) -> Option { + direct.windows(2).position(|window| { + tokens[window[0]].token_type == TokenType::Group + && tokens[window[1]].text.eq_ignore_ascii_case("BY") + }) +} + +fn expression_signatures( + tokens: &[Token], + depths: &[usize], + range: &ExpressionRange, +) -> Vec { + let mut boundaries: Vec = (range.start..range.end) + .filter(|&index| { + depths[index] == range.depth && tokens[index].token_type == TokenType::Comma + }) + .collect(); + boundaries.push(range.end); + let mut segment_start = range.start; + let mut signatures: Vec = Vec::new(); + for segment_end in boundaries { + let effective_end = alias_start(tokens, depths, range, segment_start..segment_end); + let parts: Vec = tokens[segment_start..effective_end] + .iter() + .map(|token| token.text.to_ascii_lowercase()) + .collect(); + let signature = parts.join(" "); + if !signature.is_empty() { + signatures.push(signature); + } + segment_start = segment_end + 1; + } + signatures +} + +fn alias_start( + tokens: &[Token], + depths: &[usize], + range: &ExpressionRange, + segment: std::ops::Range, +) -> usize { + if !range.strip_alias { + return segment.end; + } + segment + .clone() + .find(|&index| depths[index] == range.depth && tokens[index].token_type == TokenType::As) + .unwrap_or(segment.end) +} + +fn collect_from_and_join_facts(tokens: &[Token], direct: &[usize]) -> (Vec, Vec) { + let mut implicit: Vec = Vec::new(); + let mut missing_conditions: Vec = Vec::new(); + let Some(from_position) = direct + .iter() + .position(|&index| is_query_from(tokens, index)) + else { + return (implicit, missing_conditions); + }; + let clause_end = direct[from_position + 1..] + .iter() + .position(|&index| is_post_from_clause(tokens[index].token_type)) + .map_or(direct.len(), |offset| from_position + 1 + offset); + let relation_clause = &direct[from_position + 1..clause_end]; + let values_relation = relation_clause + .first() + .is_some_and(|&index| tokens[index].token_type == TokenType::Values); + for (position, &index) in relation_clause.iter().enumerate() { + if tokens[index].token_type == TokenType::Comma + && !values_relation + && relation_clause + .get(position + 1) + .is_none_or(|&next| tokens[next].token_type != TokenType::Lateral) + { + implicit.push(tokens[index].span); + } + } + for (position, &index) in direct.iter().enumerate() { + if tokens[index].token_type != TokenType::Join + || is_conditionless_join(tokens, direct, position) + { + continue; + } + let condition_end = direct[position + 1..] + .iter() + .position(|&candidate| { + tokens[candidate].token_type == TokenType::Join + || is_post_from_clause(tokens[candidate].token_type) + }) + .map_or(direct.len(), |offset| position + 1 + offset); + let condition = &direct[position + 1..condition_end]; + let on_position = condition + .iter() + .position(|&candidate| tokens[candidate].token_type == TokenType::On); + let has_using = condition + .iter() + .any(|&candidate| tokens[candidate].token_type == TokenType::Using); + if !has_using + && on_position.is_none_or(|offset| { + let end = direct.get(condition_end).copied().unwrap_or(tokens.len()); + trivial_on_condition(tokens, &condition_tokens(tokens, condition[offset], end)) + }) + { + missing_conditions.push(tokens[index].span); + } + } + (implicit, missing_conditions) +} + +fn is_post_from_clause(token_type: TokenType) -> bool { + matches!( + token_type, + TokenType::Join + | TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + | TokenType::Fetch + ) +} + +fn is_conditionless_join(tokens: &[Token], direct: &[usize], join_position: usize) -> bool { + direct[..join_position] + .iter() + .rev() + .take_while(|&&index| { + !matches!( + tokens[index].token_type, + TokenType::From | TokenType::Join | TokenType::Comma + ) + }) + .any(|&index| { + matches!( + tokens[index].text.to_ascii_uppercase().as_str(), + "CROSS" | "NATURAL" | "APPLY" | "POSITIONAL" | "PASTE" + ) + }) +} + +fn collect_set_operation_facts(tokens: &[Token], depths: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if !matches!( + token.token_type, + TokenType::Union | TokenType::Intersect | TokenType::Except + ) { + continue; + } + let depth = depths[index]; + let by_name = tokens[index + 1..] + .iter() + .take_while(|candidate| candidate.token_type != TokenType::Select) + .any(|candidate| candidate.text.eq_ignore_ascii_case("BY")) + && tokens[index + 1..] + .iter() + .take_while(|candidate| candidate.token_type != TokenType::Select) + .any(|candidate| candidate.text.eq_ignore_ascii_case("NAME")); + if !by_name + && (select_before_projects_star(tokens, depths, index, depth) + || select_after_projects_star(tokens, depths, index, depth)) + { + spans.push(token.span); + } + } + spans +} + +fn select_before_projects_star( + tokens: &[Token], + depths: &[usize], + end: usize, + depth: usize, +) -> bool { + let Some(start) = (0..end) + .rev() + .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) + .or_else(|| { + (0..end) + .rev() + .filter(|&index| tokens[index].token_type == TokenType::Select) + .min_by_key(|&index| depths[index]) + }) + else { + return false; + }; + projection_projects_star( + tokens, + depths, + &ExpressionRange { + start, + end, + depth: depths[start], + strip_alias: false, + }, + ) +} + +fn select_after_projects_star( + tokens: &[Token], + depths: &[usize], + start: usize, + depth: usize, +) -> bool { + let Some(select) = (start + 1..tokens.len()) + .find(|&index| depths[index] == depth && tokens[index].token_type == TokenType::Select) + .or_else(|| { + (start + 1..tokens.len()) + .filter(|&index| tokens[index].token_type == TokenType::Select) + .min_by_key(|&index| depths[index]) + }) + else { + return false; + }; + let select_depth = depths[select]; + projection_projects_star( + tokens, + depths, + &ExpressionRange { + start: select, + end: query_end(tokens, depths, select, select_depth), + depth: select_depth, + strip_alias: false, + }, + ) +} + +fn projection_projects_star(tokens: &[Token], depths: &[usize], range: &ExpressionRange) -> bool { + (range.start + 1..range.end) + .take_while(|&index| depths[index] != range.depth || !is_query_from(tokens, index)) + .any(|index| depths[index] == range.depth && tokens[index].token_type == TokenType::Star) +} + +fn collect_unused_ctes(expression: &Expression, options: &FactBuildOptions<'_>) -> Vec { + let mut unused: Vec = Vec::new(); + for node in expression.dfs() { + let with: Option<&With> = match node { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(operation) => operation.with.as_ref(), + Expression::Intersect(operation) => operation.with.as_ref(), + Expression::Except(operation) => operation.with.as_ref(), + _ => None, + }; + if let Some(with) = with { + unused.extend(collect_unused_for_with( + with, + &outside_with_table_names(node, with), + options, + )); + } + } + unused +} + +/// Table names `root` references outside its own `with`, counted so a name the WITH also uses still counts. +fn outside_with_table_names(root: &Expression, with: &With) -> HashSet { + let mut counts: HashMap = HashMap::new(); + for name in table_name_occurrences(root) { + *counts.entry(name).or_default() += 1; + } + let inside = with + .ctes + .iter() + .map(|cte| &cte.this) + .chain(with.search.as_deref()); + for expression in inside { + for name in table_name_occurrences(expression) { + if let Some(count) = counts.get_mut(&name) { + *count -= 1; + } + } + } + counts + .into_iter() + .filter_map(|(name, count)| (count > 0).then_some(name)) + .collect() +} + +fn collect_unused_for_with( + with: &With, + root_tables: &HashSet, + options: &FactBuildOptions<'_>, +) -> Vec { + let names: HashSet = with + .ctes + .iter() + .map(|cte| cte.alias.name.to_ascii_lowercase()) + .collect(); + let dependencies: HashMap> = with + .ctes + .iter() + .map(|cte| { + ( + cte.alias.name.to_ascii_lowercase(), + table_names(&cte.this) + .intersection(&names) + .cloned() + .collect(), + ) + }) + .collect(); + let mut reachable: HashSet = root_tables.intersection(&names).cloned().collect(); + reachable.extend( + options + .externally_referenced_ctes + .intersection(&names) + .cloned(), + ); + if let Some(harness) = options.fixture_harness { + reachable.extend( + names + .iter() + .filter(|name| is_harness_cte(harness, name)) + .cloned(), + ); + } + let mut queue: VecDeque = reachable.iter().cloned().collect(); + while let Some(name) = queue.pop_front() { + if let Some(required) = dependencies.get(&name) { + for dependency in required { + if reachable.insert(dependency.clone()) { + queue.push_back(dependency.clone()); + } + } + } + } + with.ctes + .iter() + .filter(|cte| !reachable.contains(&cte.alias.name.to_ascii_lowercase())) + .map(|cte| cte.alias.name.clone()) + .collect() +} + +fn table_names(expression: &Expression) -> HashSet { + table_name_occurrences(expression).collect() +} + +fn table_name_occurrences(expression: &Expression) -> impl Iterator + '_ { + expression.dfs().filter_map(|node| match node { + Expression::Table(table) => Some(table.name.name.to_ascii_lowercase()), + _ => None, + }) +} + +fn null_comparison_fix(tokens: &[Token], span: Span) -> Option { + let index = token_index_for_span(tokens, span)?; + let token = &tokens[index]; + if !adjacent_null(tokens, index, true) { + return None; + } + let replacement = match token.token_type { + TokenType::Eq => "IS", + TokenType::Neq => "IS NOT", + _ => return None, + }; + Some(LintEdit { + start: span.start, + end: span.end, + replacement: replacement.to_string(), + }) +} + +fn redundant_distinct_fix(tokens: &[Token], span: Span) -> LintEdit { + let end = token_index_for_span(tokens, span) + .and_then(|index| significant_after(tokens, index)) + .filter(|&index| !is_comment(&tokens[index])) + .map_or(span.end, |index| tokens[index].span.start); + LintEdit { + start: span.start, + end, + replacement: String::new(), + } +} + +fn implicit_cartesian_fix(sql: &str, tokens: &[Token], span: Span) -> Option { + let comma_index = token_index_for_span(tokens, span)?; + let depths = token_depths(tokens); + let depth = depths[comma_index]; + let from_index = (0..comma_index).rev().find(|&index| { + depths[index] == depth + && (is_query_from(tokens, index) || tokens[index].token_type == TokenType::Semicolon) + })?; + if !is_query_from(tokens, from_index) { + return None; + } + let clause_end = (comma_index + 1..tokens.len()) + .find(|&index| depths[index] == depth && is_post_from_clause(tokens[index].token_type)) + .unwrap_or(tokens.len()); + if tokens[from_index + 1..clause_end] + .iter() + .any(|token| token.token_type == TokenType::Join) + { + return None; + } + let next_index = significant_after(tokens, comma_index)?; + let end = tokens[next_index].span.start; + let replaced = char_slice(sql, span.start, end)?; + if contains_comment(replaced) { + return None; + } + Some(LintEdit { + start: span.start, + end, + replacement: " CROSS JOIN ".to_string(), + }) +} + +fn conditionless_join_fix(tokens: &[Token], span: Span) -> Option { + let join_index = token_index_for_span(tokens, span)?; + let depths = token_depths(tokens); + let depth = depths[join_index]; + if significant_before(tokens, join_index).is_some_and(|index| { + matches!( + tokens[index].text.to_ascii_uppercase().as_str(), + "INNER" + | "LEFT" + | "RIGHT" + | "FULL" + | "OUTER" + | "CROSS" + | "NATURAL" + | "ASOF" + | "SEMI" + | "ANTI" + ) + }) { + return None; + } + let condition_end = (join_index + 1..tokens.len()) + .find(|&index| { + depths[index] == depth + && (tokens[index].token_type == TokenType::Join + || is_post_from_clause(tokens[index].token_type)) + }) + .unwrap_or(tokens.len()); + if tokens[join_index + 1..condition_end] + .iter() + .any(|token| matches!(token.token_type, TokenType::On | TokenType::Using)) + { + return None; + } + Some(LintEdit { + start: span.start, + end: span.end, + replacement: "CROSS JOIN".to_string(), + }) +} + +fn unused_cte_fix(sql: &str, tokens: &[Token], name_span: Span) -> Option { + let name_index = token_index_for_span(tokens, name_span)?; + if tokens[name_index].text.starts_with("__") { + return None; + } + let as_index = significant_after(tokens, name_index)?; + if tokens[as_index].token_type != TokenType::As { + return None; + } + let open_index = significant_after(tokens, as_index)?; + if tokens[open_index].token_type != TokenType::LParen { + return None; + } + let depths = token_depths(tokens); + let close_depth = depths[open_index] + 1; + let close_index = (open_index + 1..tokens.len()).find(|&index| { + tokens[index].token_type == TokenType::RParen && depths[index] == close_depth + })?; + let body_tokens = &tokens[open_index + 1..close_index]; + let first_body = body_tokens.iter().find(|token| !is_layout(token))?; + if !matches!(first_body.token_type, TokenType::Select | TokenType::With) + || body_tokens.iter().any(|token| { + matches!( + token.text.to_ascii_uppercase().as_str(), + "INSERT" | "UPDATE" | "DELETE" | "MERGE" + ) + }) + { + return None; + } + let previous_index = significant_before(tokens, name_index)?; + let next_index = significant_after(tokens, close_index)?; + let (start, end) = if tokens[next_index].token_type == TokenType::Comma { + let following_index = significant_after(tokens, next_index)?; + (name_span.start, tokens[following_index].span.start) + } else if tokens[previous_index].token_type == TokenType::Comma { + ( + tokens[previous_index].span.start, + tokens[close_index].span.end, + ) + } else if tokens[previous_index].token_type == TokenType::With { + ( + tokens[previous_index].span.start, + tokens[next_index].span.start, + ) + } else { + return None; + }; + if contains_comment(char_slice(sql, start, end)?) { + return None; + } + Some(LintEdit { + start, + end, + replacement: String::new(), + }) +} + +fn token_index_for_span(tokens: &[Token], span: Span) -> Option { + tokens.iter().position(|token| token.span == span) +} + +pub(super) fn significant_before(tokens: &[Token], index: usize) -> Option { + (0..index) + .rev() + .find(|&candidate| !is_layout(&tokens[candidate])) +} + +pub(super) fn significant_after(tokens: &[Token], index: usize) -> Option { + (index + 1..tokens.len()).find(|&candidate| !is_layout(&tokens[candidate])) +} + +pub(super) fn is_layout(token: &Token) -> bool { + matches!(token.token_type, TokenType::Space | TokenType::Break) +} + +pub(super) fn is_comment(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::LineComment | TokenType::BlockComment + ) +} + +fn diagnostics(context: &DiagnosticContext<'_>) -> Vec { + let mut output: Vec = core_diagnostics(context); + output.extend(additional_style_diagnostics(context)); + output.extend(additional_relation_diagnostics(context)); + output.extend(rule_migration_diagnostics(context)); + output +} + +fn core_diagnostics(context: &DiagnosticContext<'_>) -> Vec { + let DiagnosticContext { + sql, + dialect, + facts, + tokens, + enabled, + } = context; + let mut diagnostics: Vec = Vec::new(); + if enabled.contains(NULL_COMPARISON.code) { + diagnostics.extend(facts.null_comparisons.iter().map(|span| { + let fix = null_comparison_fix(tokens, *span); + diagnostic( + &NULL_COMPARISON, + Some(*span), + fix, + Some("only equality and inequality comparisons have a deterministic repair"), + ) + })); + } + if enabled.contains(IMPLICIT_CARTESIAN_JOIN.code) { + diagnostics.extend(facts.implicit_cartesian_joins.iter().map(|span| { + let fix = implicit_cartesian_fix(sql, tokens, *span); + diagnostic( + &IMPLICIT_CARTESIAN_JOIN, + Some(*span), + fix, + Some("mixed joins, comments, or generated SQL require an explicit authored repair"), + ) + })); + } + if enabled.contains(JOIN_WITHOUT_CONDITION.code) { + diagnostics.extend(facts.joins_without_condition.iter().map(|span| { + let fix = conditionless_join_fix(tokens, *span); + diagnostic( + &JOIN_WITHOUT_CONDITION, + Some(*span), + fix, + Some("qualified joins or joins with placeholder conditions require user intent"), + ) + })); + } + if enabled.contains(UNORDERED_LIMIT.code) { + diagnostics.extend(diagnostics_for_spans( + &UNORDERED_LIMIT, + &facts.unordered_limits, + Some("deterministic ordering columns and tie-breakers require user intent"), + )); + } + if enabled.contains(UNUSED_CTE.code) { + for span in unused_cte_spans(tokens, &facts.unused_cte_names) { + let fix = unused_cte_fix(sql, tokens, span); + diagnostics.push(diagnostic( + &UNUSED_CTE, + Some(span), + fix, + Some("only contiguous authored, comment-free, select-only CTEs can be removed"), + )); + } + } + if enabled.contains(REDUNDANT_DISTINCT.code) { + diagnostics.extend(facts.redundant_distincts.iter().map(|span| { + diagnostic( + &REDUNDANT_DISTINCT, + Some(*span), + Some(redundant_distinct_fix(tokens, *span)), + None, + ) + })); + } + if enabled.contains(POSITIONAL_SET_STAR.code) { + diagnostics.extend(diagnostics_for_spans( + &POSITIONAL_SET_STAR, + &facts.positional_set_stars, + Some( + "column names and their intended positional order require resolved schema evidence", + ), + )); + } + if enabled.contains(EXPLICIT_UNION.code) { + diagnostics.extend(facts.additional.bare_unions.iter().map(|span| { + let fix = explicit_union_fix(*dialect, *span); + diagnostic( + &EXPLICIT_UNION, + Some(*span), + fix, + Some("the active dialect does not accept explicit UNION DISTINCT"), + ) + })); + } + diagnostics +} + +fn additional_style_diagnostics(context: &DiagnosticContext<'_>) -> Vec { + let DiagnosticContext { + sql, + facts, + tokens, + enabled, + .. + } = context; + let mut diagnostics: Vec = Vec::new(); + if enabled.contains(DUPLICATE_TABLE_ALIAS.code) { + diagnostics.extend(diagnostics_for_spans( + &DUPLICATE_TABLE_ALIAS, + &facts.additional.duplicate_table_aliases, + Some("choosing a new relation alias requires user intent"), + )); + } + if enabled.contains(DUPLICATE_OUTPUT_ALIAS.code) { + diagnostics.extend(diagnostics_for_spans( + &DUPLICATE_OUTPUT_ALIAS, + &facts.additional.duplicate_output_aliases, + Some("choosing a stable output name requires user intent"), + )); + } + if enabled.contains(MIXED_GROUP_ORDER_REFERENCES.code) { + diagnostics.extend(diagnostics_for_spans( + &MIXED_GROUP_ORDER_REFERENCES, + &facts.additional.mixed_group_order_references, + Some("normalization requires reliable projection-name resolution"), + )); + } + if enabled.contains(REDUNDANT_ELSE_NULL.code) { + diagnostics.extend(facts.additional.redundant_else_nulls.iter().map(|span| { + diagnostic( + &REDUNDANT_ELSE_NULL, + Some(*span), + Some(deletion(*span)), + None, + ) + })); + } + if enabled.contains(PARENTHESIZED_DISTINCT.code) { + diagnostics.extend(facts.additional.parenthesized_distincts.iter().map(|span| { + diagnostic( + &PARENTHESIZED_DISTINCT, + Some(*span), + parenthesized_distinct_fix(sql, tokens, *span), + Some("the parentheses group an expression that continues after them, or hold comments"), + ) + })); + } + if enabled.contains(CONSTANT_PREDICATE.code) { + diagnostics.extend(diagnostics_for_spans( + &CONSTANT_PREDICATE, + &facts.additional.constant_predicates, + Some("removing or replacing a scaffold predicate requires user intent"), + )); + } + if enabled.contains(CONSECUTIVE_SEMICOLON.code) { + diagnostics.extend(facts.additional.consecutive_semicolons.iter().map(|span| { + diagnostic( + &CONSECUTIVE_SEMICOLON, + Some(*span), + Some(deletion(*span)), + None, + ) + })); + } + if enabled.contains(REDUNDANT_SELF_ALIAS.code) { + diagnostics.extend(facts.additional.redundant_self_aliases.iter().map(|span| { + diagnostic( + &REDUNDANT_SELF_ALIAS, + Some(*span), + None, + Some("removing the alias may change statically inferred output identity"), + ) + })); + } + if enabled.contains(COUNT_ONE.code) { + diagnostics.extend(facts.additional.count_one_literals.iter().map(|span| { + diagnostic( + &COUNT_ONE, + Some(*span), + Some(LintEdit { + start: span.start, + end: span.end, + replacement: "*".to_string(), + }), + None, + ) + })); + } + if enabled.contains(UNSTABLE_ROW_NUMBER.code) { + diagnostics.extend(diagnostics_for_spans( + &UNSTABLE_ROW_NUMBER, + &facts.additional.unstable_row_numbers, + Some("ordering columns and a stable tie-breaker require user intent"), + )); + } + diagnostics +} + +fn additional_relation_diagnostics(context: &DiagnosticContext<'_>) -> Vec { + let DiagnosticContext { + sql, + facts, + tokens, + enabled, + .. + } = context; + let mut diagnostics: Vec = Vec::new(); + if enabled.contains(NULL_NOT_IN.code) { + diagnostics.extend(diagnostics_for_spans( + &NULL_NOT_IN, + &facts.additional.null_not_in_predicates, + Some("the intended NULL and anti-join semantics require user intent"), + )); + } + if enabled.contains(SET_ARITY_MISMATCH.code) { + diagnostics.extend(diagnostics_for_spans( + &SET_ARITY_MISMATCH, + &facts.additional.set_arity_mismatches, + Some("the intended branch schema requires user intent"), + )); + } + if enabled.contains(PROJECTED_STAR.code) { + diagnostics.extend(diagnostics_for_spans( + &PROJECTED_STAR, + &facts.additional.projected_stars, + Some("column enumeration requires reliable resolved schema evidence"), + )); + } + if enabled.contains(UNALIASED_CALCULATION.code) { + diagnostics.extend(diagnostics_for_spans( + &UNALIASED_CALCULATION, + &facts.additional.unaliased_calculations, + Some("a meaningful stable output name requires user intent"), + )); + } + if enabled.contains(UNUSED_TABLE_ALIAS.code) { + diagnostics.extend(facts.additional.unused_table_aliases.iter().map(|span| { + diagnostic( + &UNUSED_TABLE_ALIAS, + Some(*span), + Some(deletion(*span)), + None, + ) + })); + } + if enabled.contains(NULL_REJECTED_LEFT_JOIN.code) { + diagnostics.extend(diagnostics_for_spans( + &NULL_REJECTED_LEFT_JOIN, + &facts.additional.null_rejected_left_joins, + Some("moving the predicate or changing join type requires user intent"), + )); + } + if enabled.contains(IMPLICIT_INNER_JOIN.code) { + diagnostics.extend(facts.additional.implicit_inner_joins.iter().map(|span| { + diagnostic( + &IMPLICIT_INNER_JOIN, + Some(*span), + Some(LintEdit { + start: span.start, + end: span.end, + replacement: "INNER JOIN".to_string(), + }), + None, + ) + })); + } + if enabled.contains(AMBIGUOUS_ORDER_DIRECTIONS.code) { + diagnostics.extend(diagnostics_for_spans( + &AMBIGUOUS_ORDER_DIRECTIONS, + &facts.additional.ambiguous_order_directions, + Some("normalizing every expression requires a multi-range authored rewrite"), + )); + } + if enabled.contains(UNQUALIFIED_MULTI_SOURCE_COLUMN.code) { + diagnostics.extend(diagnostics_for_spans( + &UNQUALIFIED_MULTI_SOURCE_COLUMN, + &facts.additional.unqualified_multi_source_columns, + Some("choosing the correct relation qualifier requires user intent"), + )); + } + if enabled.contains(INCONSISTENT_SINGLE_SOURCE_QUALIFICATION.code) { + diagnostics.extend(diagnostics_for_spans( + &INCONSISTENT_SINGLE_SOURCE_QUALIFICATION, + &facts.additional.inconsistent_single_source_qualification, + Some("qualification style is normalized by the canonical formatter only when safe"), + )); + } + if enabled.contains(UNKNOWN_RELATION_QUALIFIER.code) { + diagnostics.extend(diagnostics_for_spans( + &UNKNOWN_RELATION_QUALIFIER, + &facts.additional.unknown_relation_qualifiers, + Some("the intended relation or alias requires user intent"), + )); + } + if enabled.contains(SIMPLE_BOOLEAN_CASE.code) { + diagnostics.extend(facts.additional.simple_boolean_cases.iter().map(|span| { + diagnostic( + &SIMPLE_BOOLEAN_CASE, + Some(*span), + simple_boolean_case_fix(sql, tokens, *span), + Some("comments or a non-canonical CASE shape prevent a lossless rewrite"), + ) + })); + } + if enabled.contains(NESTED_ELSE_CASE.code) { + diagnostics.extend(diagnostics_for_spans( + &NESTED_ELSE_CASE, + &facts.additional.nested_else_cases, + Some("flattening nested branches requires a multi-range authored rewrite"), + )); + } + if enabled.contains(UNUSED_JOINED_RELATION.code) { + diagnostics.extend(diagnostics_for_spans( + &UNUSED_JOINED_RELATION, + &facts.additional.unused_joined_relations, + Some("the join may intentionally filter or multiply rows"), + )); + } + diagnostics +} + +fn rule_migration_diagnostics(context: &DiagnosticContext<'_>) -> Vec { + let DiagnosticContext { facts, enabled, .. } = context; + let mut diagnostics: Vec = Vec::new(); + if enabled.contains(COMMENT_ATTACHMENT.code) { + diagnostics.extend(diagnostics_for_spans( + &COMMENT_ATTACHMENT, + &facts.invalid_comment_attachments, + Some("comment attachment requires an authored placement decision"), + )); + } + if enabled.contains(CTE_ONLY_BODY.code) { + diagnostics.extend(diagnostics_for_spans( + &CTE_ONLY_BODY, + &facts.cte_only_bodies, + Some("extracting transformation logic into named CTEs requires user intent"), + )); + } + if enabled.contains(TERMINAL_SELECT.code) { + diagnostics.extend(diagnostics_for_spans( + &TERMINAL_SELECT, + &facts.terminal_selects, + Some("choosing the final logical CTE and projection requires user intent"), + )); + } + if enabled.contains(NESTED_CTE.code) { + diagnostics.extend(diagnostics_for_spans( + &NESTED_CTE, + &facts.nested_ctes, + Some("hoisting a nested CTE requires an authored query rewrite"), + )); + } + if enabled.contains(RECURSIVE_CTE.code) { + diagnostics.extend(diagnostics_for_spans( + &RECURSIVE_CTE, + &facts.recursive_ctes, + Some("replacing recursion requires an authored query design"), + )); + } + if enabled.contains(INLINE_QUERY_RELATION.code) { + diagnostics.extend(diagnostics_for_spans( + &INLINE_QUERY_RELATION, + &facts.inline_query_relations, + Some("extracting a derived relation can change correlation and scope"), + )); + } + diagnostics +} + +fn deletion(span: Span) -> LintEdit { + LintEdit { + start: span.start, + end: span.end, + replacement: String::new(), + } +} + +/// `DISTINCT `, only when the parentheses hold the whole select item, so removing them cannot regroup an operator. +fn parenthesized_distinct_fix(sql: &str, tokens: &[Token], span: Span) -> Option { + let source = char_slice(sql, span.start, span.end)?; + let close = tokens.iter().position(|token| token.span.end == span.end)?; + let item_ends = significant_after(tokens, close) + .is_none_or(|next| SELECT_ITEM_END_TOKENS.contains(&tokens[next].token_type)); + if contains_comment(source) || !item_ends { + return None; + } + let open = source.find('(')?; + let close = source.rfind(')')?; + (open < close).then(|| LintEdit { + start: span.start, + end: span.end, + replacement: format!("DISTINCT {}", &source[open + 1..close]), + }) +} + +fn simple_boolean_case_fix(sql: &str, tokens: &[Token], span: Span) -> Option { + let case_index = token_index_for_span(tokens, span).or_else(|| { + tokens + .iter() + .position(|token| token.span.start == span.start && token.token_type == TokenType::Case) + })?; + let end_index = tokens.iter().position(|token| token.span.end == span.end)?; + let source = char_slice(sql, span.start, span.end)?; + if contains_comment(source) { + return None; + } + let case_tokens = &tokens[case_index..=end_index]; + if case_tokens + .iter() + .filter(|token| token.token_type == TokenType::When) + .count() + != 1 + || case_tokens + .iter() + .filter(|token| token.token_type == TokenType::Then) + .count() + != 1 + || case_tokens + .iter() + .filter(|token| token.token_type == TokenType::Else) + .count() + != 1 + { + return None; + } + let when_index = + (case_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::When)?; + let then_index = + (when_index + 1..end_index).find(|&index| tokens[index].token_type == TokenType::Then)?; + let condition = char_slice( + sql, + tokens[when_index].span.end, + tokens[then_index].span.start, + )? + .trim(); + (!condition.is_empty()).then(|| LintEdit { + start: span.start, + end: span.end, + replacement: format!("COALESCE({condition}, FALSE)"), + }) +} + +fn explicit_union_fix(dialect: Dialect, span: Span) -> Option { + matches!( + dialect, + Dialect::PostgreSQL | Dialect::Snowflake | Dialect::DuckDB + ) + .then(|| LintEdit { + start: span.start, + end: span.end, + replacement: "UNION DISTINCT".to_string(), + }) +} + +fn char_slice(sql: &str, start: usize, end: usize) -> Option<&str> { + if start > end { + return None; + } + let byte_start = char_offset_to_byte_offset(sql, start)?; + let byte_end = char_offset_to_byte_offset(sql, end)?; + sql.get(byte_start..byte_end) +} + +fn char_offset_to_byte_offset(sql: &str, offset: usize) -> Option { + if offset == sql.chars().count() { + return Some(sql.len()); + } + sql.char_indices() + .nth(offset) + .map(|(byte_offset, _)| byte_offset) +} + +fn contains_comment(source: &str) -> bool { + source.contains("--") || source.contains("/*") +} + +fn diagnostics_for_spans( + rule: &LintRuleMetadata, + spans: &[Span], + fix_unavailable_reason: Option<&'static str>, +) -> Vec { + spans + .iter() + .map(|span| diagnostic(rule, Some(*span), None, fix_unavailable_reason)) + .collect() +} + +fn diagnostic( + rule: &LintRuleMetadata, + span: Option, + fix: Option, + fix_unavailable_reason: Option<&'static str>, +) -> LintDiagnostic { + let span = span.unwrap_or_default(); + LintDiagnostic { + code: rule.code, + message: Cow::Borrowed(rule.message), + remediation: Cow::Borrowed(rule.remediation), + start: span.start, + end: span.end, + fix_unavailable_reason: fix.is_none().then_some(fix_unavailable_reason).flatten(), + fix, + } +} + +fn unused_cte_spans(tokens: &[Token], unused_names: &[String]) -> Vec { + let mut remaining: Vec = unused_names + .iter() + .map(|name| name.to_ascii_lowercase()) + .collect(); + let mut spans: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + let name = token.text.to_ascii_lowercase(); + let Some(position) = remaining.iter().position(|candidate| candidate == &name) else { + continue; + }; + if tokens + .get(index + 1) + .is_some_and(|next| next.token_type == TokenType::As) + { + remaining.remove(position); + spans.push(token.span); + } + } + spans +} + +fn null_comparison_spans(tokens: &[Token]) -> Vec { + let comparison_types = [ + TokenType::Eq, + TokenType::Neq, + TokenType::Lt, + TokenType::Lte, + TokenType::Gt, + TokenType::Gte, + ]; + let depths = token_depths(tokens); + let assignments: Vec = assignment_operators(tokens, &depths); + tokens + .iter() + .enumerate() + .filter(|(_, token)| comparison_types.contains(&token.token_type)) + .filter(|(index, _)| !assignments[*index]) + .filter(|(index, _)| { + adjacent_null(tokens, *index, false) || adjacent_null(tokens, *index, true) + }) + .map(|(_, token)| token.span) + .collect() +} + +/// For each token, whether its nearest same-depth clause opener is `SET`, found in one forward pass. +fn assignment_operators(tokens: &[Token], depths: &[usize]) -> Vec { + let mut in_set: Vec = Vec::new(); + let mut assignments: Vec = Vec::with_capacity(tokens.len()); + for (token, &depth) in tokens.iter().zip(depths) { + if in_set.len() <= depth { + in_set.resize(depth + 1, false); + } + assignments.push(in_set[depth]); + match token.token_type { + TokenType::Set => in_set[depth] = true, + TokenType::Where + | TokenType::Having + | TokenType::Qualify + | TokenType::On + | TokenType::Select + | TokenType::Semicolon => in_set[depth] = false, + _ => {} + } + } + assignments +} + +fn adjacent_null(tokens: &[Token], operator_index: usize, forward: bool) -> bool { + let mut index = operator_index as isize + if forward { 1 } else { -1 }; + let mut wrappers = 0_usize; + loop { + if index < 0 { + return false; + } + let Some(token) = tokens.get(index as usize) else { + return false; + }; + if token.token_type == TokenType::Null { + break; + } + let wrapper = if forward { + TokenType::LParen + } else { + TokenType::RParen + }; + if token.token_type != wrapper { + return false; + } + wrappers += 1; + index += if forward { 1 } else { -1 }; + } + index += if forward { 1 } else { -1 }; + let closing_wrapper = if forward { + TokenType::RParen + } else { + TokenType::LParen + }; + for _ in 0..wrappers { + if index < 0 { + return false; + } + let Some(token) = tokens.get(index as usize) else { + return false; + }; + if token.token_type != closing_wrapper { + return false; + } + index += if forward { 1 } else { -1 }; + } + true +} diff --git a/crates/sqlquack-lint/src/lint/_helpers/inline_relations.rs b/crates/sqlquack-lint/src/lint/_helpers/inline_relations.rs new file mode 100644 index 000000000..18edecf12 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/inline_relations.rs @@ -0,0 +1,223 @@ +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; + +use crate::lint::_helpers::engine::is_query_from; + +struct InlineRelationContext<'a> { + tokens: &'a [Token], + depths: &'a [usize], + scopes: Vec>, + significant: &'a [usize], +} + +pub(super) fn collect_inline_query_relation_spans( + tokens: &[Token], + depths: &[usize], + significant: &[usize], +) -> Vec { + let context = InlineRelationContext { + tokens, + depths, + scopes: token_scopes(tokens), + significant, + }; + (0..significant.len()) + .filter_map(|position| inline_query_relation_span(&context, position)) + .collect() +} + +fn inline_query_relation_span( + context: &InlineRelationContext<'_>, + position: usize, +) -> Option { + let &opening = context.significant.get(position)?; + if context.tokens[opening].token_type != TokenType::LParen { + return None; + } + let marker_position = position.checked_sub(1)?; + let previous = context.significant[marker_position]; + if context.tokens[previous].token_type == TokenType::Lateral + || context.tokens[previous].text.eq_ignore_ascii_case("apply") + || !is_query_relation_marker(context, marker_position) + { + return None; + } + + let mut candidate_position = position + 1; + let mut candidate_depth = context.depths[opening] + 1; + loop { + let &candidate = context.significant.get(candidate_position)?; + if context.depths[candidate] != candidate_depth { + return None; + } + if context.tokens[candidate].token_type == TokenType::LParen { + candidate_position += 1; + candidate_depth += 1; + continue; + } + return is_query_start(&context.tokens[candidate]) + .then_some(context.tokens[candidate].span); + } +} + +fn is_query_relation_marker(context: &InlineRelationContext<'_>, marker_position: usize) -> bool { + is_query_relation_marker_base(context, marker_position) + && !query_or_ancestor_is_ignored(context, marker_position) +} + +fn is_query_relation_marker_base( + context: &InlineRelationContext<'_>, + marker_position: usize, +) -> bool { + let marker = context.significant[marker_position]; + if !(is_query_from(context.tokens, marker) + || matches!( + context.tokens[marker].token_type, + TokenType::Join | TokenType::Comma + )) + { + return false; + } + let scope = context.scopes[marker]; + let Some(select_position) = (0..marker_position).rev().find(|&position| { + let index = context.significant[position]; + context.scopes[index] == scope && context.tokens[index].token_type == TokenType::Select + }) else { + return false; + }; + let query_clause = &context.significant[select_position + 1..marker_position]; + if query_clause.iter().any(|&index| { + context.scopes[index] == scope + && matches!( + context.tokens[index].token_type, + TokenType::Semicolon | TokenType::Union | TokenType::Intersect | TokenType::Except + ) + }) { + return false; + } + if is_query_from(context.tokens, marker) { + return true; + } + let Some(from_position) = query_clause + .iter() + .position(|&index| context.scopes[index] == scope && is_query_from(context.tokens, index)) + else { + return false; + }; + !query_clause[from_position + 1..].iter().any(|&index| { + context.scopes[index] == scope && is_post_relation_clause(context.tokens[index].token_type) + }) +} + +fn query_or_ancestor_is_ignored( + context: &InlineRelationContext<'_>, + marker_position: usize, +) -> bool { + let marker = context.significant[marker_position]; + let mut scope = context.scopes[marker]; + while let Some(opening) = scope { + if scope_starts_query(context, opening) && query_scope_is_ignored(context, opening) { + return true; + } + scope = context.scopes[opening]; + } + false +} + +fn scope_starts_query(context: &InlineRelationContext<'_>, opening: usize) -> bool { + let Ok(mut position) = context.significant.binary_search(&opening) else { + return false; + }; + position += 1; + let mut expected_depth = context.depths[opening] + 1; + loop { + let Some(&candidate) = context.significant.get(position) else { + return false; + }; + if context.depths[candidate] != expected_depth { + return false; + } + if context.tokens[candidate].token_type == TokenType::LParen { + position += 1; + expected_depth += 1; + continue; + } + return is_query_start(&context.tokens[candidate]); + } +} + +fn query_scope_is_ignored(context: &InlineRelationContext<'_>, opening: usize) -> bool { + let Ok(mut opening_position) = context.significant.binary_search(&opening) else { + return false; + }; + while opening_position > 0 + && context.tokens[context.significant[opening_position - 1]].token_type == TokenType::LParen + { + opening_position -= 1; + } + let Some(marker_position) = opening_position.checked_sub(1) else { + return false; + }; + let marker = context.significant[marker_position]; + match context.tokens[marker].token_type { + TokenType::Exists | TokenType::In => true, + TokenType::From | TokenType::Join | TokenType::As => false, + TokenType::Comma => !is_query_relation_marker_base(context, marker_position), + TokenType::Union + | TokenType::Intersect + | TokenType::Except + | TokenType::All + | TokenType::Distinct + | TokenType::Semicolon + | TokenType::Lateral => false, + _ if context.tokens[marker].text.eq_ignore_ascii_case("apply") + || context.tokens[marker] + .text + .eq_ignore_ascii_case("materialized") => + { + false + } + _ => true, + } +} + +fn is_query_start(token: &Token) -> bool { + matches!(token.token_type, TokenType::Select | TokenType::Values) + || token.text.eq_ignore_ascii_case("with") +} + +fn is_post_relation_clause(token_type: TokenType) -> bool { + matches!( + token_type, + TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + | TokenType::Fetch + ) +} + +fn token_scopes(tokens: &[Token]) -> Vec> { + let mut openings: Vec = Vec::new(); + tokens + .iter() + .enumerate() + .map(|(index, token)| { + let scope = openings.last().copied(); + if matches!( + token.token_type, + TokenType::LParen | TokenType::LBracket | TokenType::LBrace + ) { + openings.push(index); + } else if matches!( + token.token_type, + TokenType::RParen | TokenType::RBracket | TokenType::RBrace + ) { + let _ = openings.pop(); + } + scope + }) + .collect() +} diff --git a/crates/sqlquack-lint/src/lint/_helpers/join_predicates.rs b/crates/sqlquack-lint/src/lint/_helpers/join_predicates.rs new file mode 100644 index 000000000..e78c336ac --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/join_predicates.rs @@ -0,0 +1,777 @@ +//! Join predicates, and whether an explicit cross join can multiply rows. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::Literal; +use sqlquack_syntax::expressions::names::models::TableRef; +use sqlquack_syntax::expressions::queries::models::JoinKind; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::expressions::queries::models::Subquery; +use sqlquack_syntax::expressions::queries::models::With; +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; +use sqlquack_syntax::{Dialect, ExpressionWalk, parse_expressions}; + +use crate::constants::SETTING_SNIPPET_INDENT; +use crate::lint::_helpers::engine::{is_comment, is_layout}; +use crate::lint::constants::{ + CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PATH_PLACEHOLDER, CROSS_JOIN_PREFIX, + CROSS_JOIN_REASON_PLACEHOLDER, CROSS_JOIN_SUFFIX, CROSS_JOIN_UNNAMED_MESSAGE, +}; +use crate::lint::models::{CrossJoinRequest, LintDiagnostic, LintRuleMetadata}; + +const CROSS_JOIN_FIX_REASON: &str = "a keyed relationship requires user intent"; + +/// Tokens in a constant `ON` condition such as `1 = 1`. +const TRIVIAL_EQUALITY_TOKEN_COUNT: usize = 3; +const FIRST_ROW: &str = "1"; +const ROW_GENERATOR_FUNCTIONS: [&str; 16] = [ + "unnest", + "explode", + "explode_outer", + "posexplode", + "posexplode_outer", + "inline", + "inline_outer", + "flatten", + "generate_series", + "generate_subscripts", + "json_each", + "json_tree", + "json_array_elements", + "jsonb_array_elements", + "range", + "stack", +]; + +pub(super) fn join_expression_spans( + tokens: &[Token], + depths: &[usize], + dialect: Dialect, +) -> Vec { + let mut spans: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if token.token_type != TokenType::On { + continue; + } + let clause = (0..index).rev().find(|&previous| { + depths[previous] == depths[index] + && matches!( + tokens[previous].token_type, + TokenType::Join + | TokenType::Select + | TokenType::Where + | TokenType::From + | TokenType::On + ) + }); + if clause.is_none_or(|previous| tokens[previous].token_type != TokenType::Join) { + continue; + } + let end = (index + 1..tokens.len()) + .find(|&next| { + depths[next] < depths[index] + || (depths[next] == depths[index] + && matches!( + tokens[next].token_type, + TokenType::Join + | TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + | TokenType::Semicolon + )) + }) + .unwrap_or(tokens.len()); + let parsed = parse_expressions(tokens[index + 1..end].to_vec(), dialect); + if !matches!(parsed.as_deref(), Ok([predicate]) if plain_predicate(predicate)) { + spans.push(token.span); + } + } + spans +} + +fn plain_column(expression: &Expression) -> bool { + matches!( + expression, + Expression::Column(_) | Expression::Identifier(_) + ) || matches!(expression, Expression::Paren(value) if plain_column(&value.this)) +} + +fn literal(expression: &Expression) -> bool { + matches!( + expression, + Expression::Literal(_) | Expression::Boolean(_) | Expression::Null(_) + ) || matches!(expression, Expression::Neg(value) if matches!(&value.this, Expression::Literal(_))) +} + +fn operand(expression: &Expression) -> bool { + plain_column(expression) || literal(expression) +} + +fn comparison(left: &Expression, right: &Expression) -> bool { + operand(left) && operand(right) +} + +/// Whether every comparison joined by AND, OR and parentheses compares plain operands, read without recursion. +fn plain_predicate(expression: &Expression) -> bool { + let mut pending: Vec<&Expression> = vec![expression]; + while let Some(node) = pending.pop() { + match node { + Expression::Paren(value) => pending.push(&value.this), + Expression::And(value) | Expression::Or(value) => { + pending.extend([&value.right, &value.left]); + } + _ if !plain_comparison(node) => return false, + _ => {} + } + } + true +} + +fn plain_comparison(expression: &Expression) -> bool { + match expression { + Expression::Boolean(_) => true, + Expression::Eq(value) + | Expression::Neq(value) + | Expression::Lt(value) + | Expression::Lte(value) + | Expression::Gt(value) + | Expression::Gte(value) + | Expression::NullSafeEq(value) + | Expression::NullSafeNeq(value) + | Expression::Adjacent(value) + | Expression::ArrayContainsAll(value) + | Expression::ArrayContainedBy(value) + | Expression::ArrayOverlaps(value) + | Expression::ExtendsLeft(value) + | Expression::ExtendsRight(value) + | Expression::Match(value) + | Expression::TsMatch(value) => comparison(&value.left, &value.right), + Expression::Like(value) | Expression::ILike(value) => { + comparison(&value.left, &value.right) + && value.escape.as_ref().is_none_or(literal) + && value.quantifier.is_none() + } + Expression::Between(value) => { + plain_column(&value.this) && operand(&value.low) && operand(&value.high) + } + Expression::In(value) => { + plain_column(&value.this) + && value.query.is_none() + && value.unnest.is_none() + && !value.is_field + && !value.expressions.is_empty() + && value.expressions.iter().all(literal) + } + Expression::IsNull(value) => plain_column(&value.this), + _ => false, + } +} + +/// A `rule` diagnostic for every cross join that can multiply rows, naming its relation and CTE. +pub(super) fn cross_join_diagnostics( + request: &CrossJoinRequest<'_>, + rule: &LintRuleMetadata, +) -> Vec { + let one_row_relations: HashSet = request + .relation_keys + .iter() + .filter(|(_, keys)| keys.iter().any(Vec::is_empty)) + .map(|(name, _)| name.to_ascii_lowercase()) + .collect(); + let relation_names: HashMap = request + .relation_names + .iter() + .map(|(sentinel, name)| (sentinel.to_ascii_lowercase(), name.clone())) + .collect(); + let inputs = OneRowInputs { + one_row_relations: &one_row_relations, + relation_names: &relation_names, + }; + multiplying_cross_joins( + request.statements, + request.tokens, + request.cross_spans, + &inputs, + ) + .into_iter() + .map(|finding| LintDiagnostic { + code: rule.code, + message: finding + .relation + .as_deref() + .map_or(Cow::Borrowed(CROSS_JOIN_UNNAMED_MESSAGE), |relation| { + Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) + }), + remediation: if finding.relation.is_some() { + Cow::Borrowed(rule.remediation) + } else { + Cow::Owned(file_wide_remediation(rule.code)) + }, + start: finding.span.start, + end: finding.span.end, + fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), + fix: None, + }) + .collect() +} + +/// Help for a join no relation names: fix it, or record the file-wide exception it allows. +fn file_wide_remediation(code: &str) -> String { + let entry: String = [ + "[[rules.rule_exceptions]]".to_owned(), + format!("rule = \"{code}\""), + format!("path = \"{CROSS_JOIN_PATH_PLACEHOLDER}\""), + format!("reason = \"{CROSS_JOIN_REASON_PLACEHOLDER}\""), + ] + .iter() + .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!( + "Use a keyed JOIN, or make one side provably at most one row. This join reads no named \ + relation, so a relation-level exception cannot cover it; when the cartesian product is \ + intentional, record a reviewed file-wide exception in the configuration file:{entry}" + ) +} + +/// The finding message naming the joined relation and, inside a CTE, that CTE. +fn cross_join_message(relation: &str, cte: Option<&str>) -> String { + match cte { + Some(cte) => { + format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_CTE_MARKER}{cte}{CROSS_JOIN_SUFFIX}") + } + None => format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_SUFFIX}"), + } +} + +/// One explicit cross join found in the query tree. +struct CrossJoinTarget { + /// Character offsets where the joined relation may start in the token stream. + anchors: Vec, + /// Whether both joined sides may hold more than one row. + multiplies: bool, + /// The joined relation's name as authored, with dependency sentinels resolved. + relation: Option, + /// The top-level CTE whose query contains the join. + cte: Option, +} + +/// The cross join reported at one `CROSS` keyword. +struct CrossJoinFinding { + span: Span, + relation: Option, + cte: Option, +} + +/// What the analysis knows about relations named in the query. +struct OneRowInputs<'a> { + /// Lowercased dependency sentinels whose relation holds at most one row. + one_row_relations: &'a HashSet, + /// Lowercased dependency sentinels mapped to the relation names they stand for. + relation_names: &'a HashMap, +} + +/// One step of `Analysis::visit`: a node with its CTE label and top-level flag, or a select's joins. +enum Visit<'a> { + Node(&'a Expression, Option<&'a str>, bool), + Joins(&'a Select, Option<&'a str>), +} + +struct Analysis<'a> { + inputs: &'a OneRowInputs<'a>, + ctes: HashMap, + targets: Vec, +} + +/// Cross joins at `cross_spans` that can multiply rows, each with its relation and CTE. +fn multiplying_cross_joins( + statements: &[Expression], + tokens: &[Token], + cross_spans: &[Span], + inputs: &OneRowInputs<'_>, +) -> Vec { + if cross_spans.is_empty() { + return Vec::new(); + } + let mut analysis = Analysis { + inputs, + ctes: HashMap::new(), + targets: Vec::new(), + }; + for statement in statements { + analysis.record_ctes(statement); + } + for statement in statements { + analysis.visit(statement); + } + let significant: Vec = (0..tokens.len()) + .filter(|&index| { + !matches!( + tokens[index].token_type, + TokenType::Space + | TokenType::Break + | TokenType::LineComment + | TokenType::BlockComment + ) + }) + .collect(); + let mut findings: Vec = Vec::new(); + for span in cross_spans { + let candidates: Vec = relation_anchor_candidates(tokens, &significant, *span); + let target: Option<&CrossJoinTarget> = analysis + .targets + .iter() + .find(|target| anchored_at(target, &candidates)); + match target { + Some(target) if !target.multiplies => {} + Some(target) => findings.push(CrossJoinFinding { + span: *span, + relation: target.relation.clone(), + cte: target.cte.clone(), + }), + None => findings.push(CrossJoinFinding { + span: *span, + relation: None, + cte: None, + }), + } + } + findings +} + +impl Analysis<'_> { + /// Record whether every CTE holds at most one row, in definition order. + fn record_ctes(&mut self, expression: &Expression) { + for node in expression.dfs() { + let Some(with) = with_clause(node) else { + continue; + }; + for cte in &with.ctes { + let one_row: bool = !with.recursive && self.at_most_one_row(&cte.this); + let name: String = cte.alias.name.to_ascii_lowercase(); + let known: bool = self.ctes.get(&name).copied().unwrap_or(one_row); + self.ctes.insert(name, known && one_row); + } + } + } + + /// Record joins under `statement` by CTE label, from a worklist so long chains cannot overflow. + fn visit(&mut self, statement: &Expression) { + let mut pending: Vec> = vec![Visit::Node(statement, None, true)]; + while let Some(item) = pending.pop() { + let (expression, cte, top) = match item { + Visit::Joins(select, cte) => { + self.record_joins(select, cte); + continue; + } + Visit::Node(expression, cte, top) => (expression, cte, top), + }; + let mut bodies: Vec<&Expression> = Vec::new(); + let mut next: Vec> = Vec::new(); + if let Some(with) = with_clause(expression) { + for definition in &with.ctes { + bodies.push(&definition.this); + let label: Option<&str> = if top { + Some(&definition.alias.name) + } else { + cte + }; + next.push(Visit::Node(&definition.this, label, false)); + } + } + if let Expression::Select(select) = expression { + next.push(Visit::Joins(select, cte)); + } + let branches_are_top: bool = top && is_set_operation(expression); + for child in expression.children() { + if bodies.iter().any(|body| std::ptr::eq(*body, child)) { + continue; + } + next.push(Visit::Node(child, cte, branches_are_top)); + } + pending.extend(next.into_iter().rev()); + } + } + + fn record_joins(&mut self, select: &Select, cte: Option<&str>) { + let single_from: Option<&Expression> = select + .from + .as_ref() + .filter(|from| from.expressions.len() == 1) + .and_then(|from| from.expressions.first()); + for (index, join) in select.joins.iter().enumerate() { + if join.kind != JoinKind::Cross { + continue; + } + let left_one_row: bool = + index == 0 && single_from.is_some_and(|from| self.relation_one_row(from)); + let multiplies: bool = !(left_one_row || self.relation_one_row(&join.this)); + self.targets.push(CrossJoinTarget { + anchors: relation_anchors(&join.this), + multiplies, + relation: self.relation_name(&join.this), + cte: cte.map(str::to_owned), + }); + } + } + + fn relation_name(&self, relation: &Expression) -> Option { + match relation { + Expression::Table(table) => Some(self.table_name(table)), + Expression::Subquery(subquery) => { + subquery.alias.as_ref().map(|alias| alias.name.clone()) + } + Expression::Alias(alias) => Some(alias.alias.name.clone()), + _ => None, + } + } + + fn table_name(&self, table: &TableRef) -> String { + if table.schema.is_none() && table.catalog.is_none() { + let lowered: String = table.name.name.to_ascii_lowercase(); + if let Some(name) = self.inputs.relation_names.get(&lowered) { + return name.clone(); + } + } + [&table.catalog, &table.schema] + .into_iter() + .flatten() + .map(|part| part.name.as_str()) + .chain([table.name.name.as_str()]) + .collect::>() + .join(".") + } + + /// Whether a query expression returns at most one row. + fn at_most_one_row(&self, expression: &Expression) -> bool { + match expression { + Expression::Select(select) => self.select_one_row(select), + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.at_most_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } + + fn subquery_one_row(&self, subquery: &Subquery) -> bool { + !subquery.lateral + && (subquery + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)) + || self.at_most_one_row(&subquery.this)) + } + + fn select_one_row(&self, select: &Select) -> bool { + if select.limit_by.is_some() || select.connect.is_some() || !select.lateral_views.is_empty() + { + return false; + } + if limited_to_one_row(select) { + return true; + } + if select.expressions.iter().any(generates_rows) { + return false; + } + if select.group_by.is_none() && select.expressions.iter().any(aggregates) { + return true; + } + if select + .qualify + .as_ref() + .is_some_and(|qualify| first_unpartitioned_row(&qualify.this, select)) + { + return true; + } + if select.group_by.is_some() || !select.joins.is_empty() { + return false; + } + match &select.from { + None => true, + Some(from) => match from.expressions.as_slice() { + [relation] => self.relation_one_row(relation), + _ => false, + }, + } + } + + /// Whether a relation in FROM or JOIN holds at most one row. + fn relation_one_row(&self, relation: &Expression) -> bool { + match relation { + Expression::Table(table) => { + if table.schema.is_some() + || table.catalog.is_some() + || table.identifier_func.is_some() + || table.changes.is_some() + { + return false; + } + let name: String = table.name.name.to_ascii_lowercase(); + match self.ctes.get(&name) { + Some(one_row) => *one_row, + None => self.inputs.one_row_relations.contains(&name), + } + } + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.relation_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } +} + +/// The WITH clause a query expression owns. +fn with_clause(expression: &Expression) -> Option<&With> { + match expression { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(union) => union.with.as_ref(), + Expression::Intersect(intersect) => intersect.with.as_ref(), + Expression::Except(except) => except.with.as_ref(), + _ => None, + } +} + +fn is_set_operation(expression: &Expression) -> bool { + matches!( + expression, + Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) + ) +} + +fn limited_to_one_row(select: &Select) -> bool { + let limited: bool = select + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)); + let fetched: bool = select.fetch.as_ref().is_some_and(|fetch| { + !fetch.percent && !fetch.with_ties && fetch.count.as_ref().is_some_and(at_most_one) + }); + let top: bool = select + .top + .as_ref() + .is_some_and(|top| !top.percent && !top.with_ties && at_most_one(&top.this)); + limited || fetched || top +} + +fn at_most_one(expression: &Expression) -> bool { + match expression { + Expression::Literal(literal) => match literal.as_ref() { + Literal::Number(text) => text.parse::().is_ok_and(|count| count <= 1), + _ => false, + }, + Expression::Paren(paren) => at_most_one(&paren.this), + _ => false, + } +} + +/// Whether a projected expression can return several rows per input row. +fn generates_rows(expression: &Expression) -> bool { + expression.dfs().any(|node| match node { + Expression::Unnest(_) + | Expression::Explode(_) + | Expression::ExplodeOuter(_) + | Expression::GenerateSeries(_) => true, + Expression::Function(function) => is_row_generator(&function.name), + _ => false, + }) +} + +fn is_row_generator(name: &str) -> bool { + let name: &str = name.rsplit('.').next().unwrap_or_default(); + ROW_GENERATOR_FUNCTIONS + .iter() + .any(|generator| generator.eq_ignore_ascii_case(name)) +} + +fn anchored_at(target: &CrossJoinTarget, candidates: &[usize]) -> bool { + target + .anchors + .iter() + .any(|anchor| candidates.contains(anchor)) +} + +/// Whether an expression aggregates outside any window function or subquery. +fn aggregates(expression: &Expression) -> bool { + let mut pending: Vec<&Expression> = vec![expression]; + while let Some(node) = pending.pop() { + if crate::tree::main::is_aggregate::is_aggregate(node) { + return true; + } + if !matches!( + node, + Expression::WindowFunction(_) | Expression::Subquery(_) | Expression::Select(_) + ) { + pending.extend(node.children()); + } + } + false +} + +/// `QUALIFY ROW_NUMBER() OVER (ORDER BY ...) = 1` without a partition, directly or by alias. +fn first_unpartitioned_row(condition: &Expression, select: &Select) -> bool { + let mut pending: Vec<&Expression> = vec![condition]; + while let Some(node) = pending.pop() { + match node { + Expression::And(operation) => pending.extend([&operation.right, &operation.left]), + Expression::Paren(paren) => pending.push(&paren.this), + _ if first_row_filter(node, select) => return true, + _ => {} + } + } + false +} + +/// One conjunct of `first_unpartitioned_row`: an unpartitioned row number compared with 1. +fn first_row_filter(condition: &Expression, select: &Select) -> bool { + match condition { + Expression::Eq(operation) => { + let compared: &Expression = match (&operation.left, &operation.right) { + (other, Expression::Literal(literal)) | (Expression::Literal(literal), other) + if matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) => + { + other + } + _ => return false, + }; + match compared { + Expression::WindowFunction(_) => unpartitioned_row_number(compared), + Expression::Column(column) if column.table.is_none() => select + .expressions + .iter() + .any(|projection| match projection { + Expression::Alias(alias) => { + alias.alias.name.eq_ignore_ascii_case(&column.name.name) + && unpartitioned_row_number(&alias.this) + } + _ => false, + }), + _ => false, + } + } + _ => false, + } +} + +fn unpartitioned_row_number(expression: &Expression) -> bool { + matches!( + expression, + Expression::WindowFunction(window) + if crate::tree::main::is_row_number::is_row_number(&window.this) + && window.over.window_name.is_none() + && window.over.partition_by.is_empty() + ) +} + +/// Where a joined relation can start: its first name part, or a derived table's alias. +fn relation_anchors(relation: &Expression) -> Vec { + match relation { + Expression::Table(table) => { + let mut anchors: Vec = Vec::new(); + for identifier in [&table.catalog, &table.schema].into_iter().flatten() { + anchors.extend(identifier.span.map(|span| span.start)); + } + anchors.extend(table.name.span.map(|span| span.start)); + anchors.extend(table.span.map(|span| span.start)); + anchors + } + Expression::Subquery(subquery) => subquery + .alias + .as_ref() + .and_then(|alias| alias.span) + .map(|span| span.start) + .into_iter() + .collect(), + Expression::Alias(alias) => alias + .alias + .span + .map(|span| span.start) + .into_iter() + .collect(), + _ => Vec::new(), + } +} + +/// Where the relation after `CROSS JOIN` at `cross` starts, and a derived table's alias. +fn relation_anchor_candidates(tokens: &[Token], significant: &[usize], cross: Span) -> Vec { + let Some(position) = significant + .iter() + .position(|&index| tokens[index].span.start == cross.start) + else { + return Vec::new(); + }; + let Some(&first) = significant.get(position + 2) else { + return Vec::new(); + }; + let mut candidates: Vec = vec![tokens[first].span.start]; + if tokens[first].token_type == TokenType::LParen { + let mut depth: usize = 0; + let mut cursor: usize = position + 2; + while let Some(&index) = significant.get(cursor) { + match tokens[index].token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth = depth.saturating_sub(1); + if depth == 0 { + break; + } + } + _ => {} + } + cursor += 1; + } + let mut alias: usize = cursor + 1; + if significant + .get(alias) + .is_some_and(|&index| tokens[index].token_type == TokenType::As) + { + alias += 1; + } + if let Some(&index) = significant.get(alias) { + candidates.push(tokens[index].span.start); + } + } + candidates +} + +/// Every token after `ON` before `end`, including those nested in parentheses, stopping at a parenthesis the condition did not open. +pub(super) fn condition_tokens(tokens: &[Token], on: usize, end: usize) -> Vec { + let mut open: usize = 0; + let mut found: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate().take(end).skip(on + 1) { + match token.token_type { + TokenType::LParen => open += 1, + TokenType::RParen if open == 0 => break, + TokenType::RParen => open -= 1, + _ => {} + } + if !is_layout(token) && !is_comment(token) { + found.push(index); + } + } + found +} + +pub(super) fn trivial_on_condition(tokens: &[Token], condition: &[usize]) -> bool { + let significant: Vec<&Token> = condition + .iter() + .map(|&index| &tokens[index]) + .filter(|token| !matches!(token.token_type, TokenType::LParen | TokenType::RParen)) + .collect(); + significant.is_empty() + || significant.len() == 1 && significant[0].token_type == TokenType::True + || significant.len() == TRIVIAL_EQUALITY_TOKEN_COUNT + && significant[1].token_type == TokenType::Eq + && matches!( + significant[0].token_type, + TokenType::Number | TokenType::String | TokenType::True | TokenType::False + ) + && significant[0].token_type == significant[2].token_type + && significant[0].text == significant[2].text +} diff --git a/crates/sqlquack-lint/src/lint/_helpers/mod.rs b/crates/sqlquack-lint/src/lint/_helpers/mod.rs new file mode 100644 index 000000000..1f156a57b --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub mod additional; +pub mod engine; +pub mod inline_relations; +pub mod join_predicates; +pub mod parsing; +pub mod terminal_shape; diff --git a/crates/sqlquack-lint/src/lint/_helpers/parsing.rs b/crates/sqlquack-lint/src/lint/_helpers/parsing.rs new file mode 100644 index 000000000..e0f1645e1 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/parsing.rs @@ -0,0 +1,102 @@ +//! The one tokenize-and-parse step every lint request shares. + +use std::collections::HashSet; + +use sqlquack_syntax::tokens::models::{Token, TokenType}; +use sqlquack_syntax::{ + ComplexityGuardOptions, Dialect, Expression, ExpressionWalk, ParseOptions, dialect_tokens, + parse_statements, +}; + +use crate::lint::constants::UNPARSED_DIALECTS; +use crate::lint::models::ParsedSql; + +const MAX_LINT_FUNCTION_CALL_DEPTH: usize = 128; + +/// The SQLQuack dialect `name` names; `None` for a known dialect SQLQuack does not parse yet. +pub(crate) fn dialect(name: &str) -> Result, String> { + let lower = name.to_ascii_lowercase(); + match lower.as_str() { + "snowflake" => Ok(Some(Dialect::Snowflake)), + "duckdb" => Ok(Some(Dialect::DuckDB)), + _ if UNPARSED_DIALECTS.contains(&lower.as_str()) => Ok(None), + _ => Err(format!("Unknown dialect: {name}")), + } +} + +/// Tokenize and parse `sql`, reading a harness's `FROM ()` as a bare relation. +pub(crate) fn parse( + sql: &str, + dialect: Dialect, + empty_relation: Option<&str>, +) -> Result { + let tokens = dialect_tokens(sql, dialect).map_err(|error| error.to_string())?; + let options = ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_function_call_depth: Some(MAX_LINT_FUNCTION_CALL_DEPTH), + ..Default::default() + }), + }; + let parse_text = empty_relation + .map(|name| without_empty_relation_parentheses(sql, &tokens, name)) + .unwrap_or_default(); + let parse_text = if parse_text.is_empty() { + sql + } else { + &parse_text + }; + let statements = + parse_statements(parse_text, dialect, &options).map_err(|error| error.to_string())?; + for statement in &statements { + if statement + .dfs() + .any(|node| matches!(node, Expression::Command(_) | Expression::Raw(_))) + { + return Err("unsupported syntax: native parser retained an opaque SQL node".to_owned()); + } + } + Ok(ParsedSql { + dialect, + tokens, + statements, + }) +} + +/// `sql` with each `FROM ( )` blanked to spaces past Snowflake's `TABLE( )` rule; empty when none. +fn without_empty_relation_parentheses(sql: &str, tokens: &[Token], empty_relation: &str) -> String { + let significant: Vec<&Token> = tokens + .iter() + .filter(|token| { + !matches!( + token.token_type, + TokenType::Space + | TokenType::Break + | TokenType::LineComment + | TokenType::BlockComment + ) + }) + .collect(); + let blanked: HashSet = significant + .windows(4) + .filter(|window| { + window[0].text.eq_ignore_ascii_case("from") + && window[1].text.eq_ignore_ascii_case(empty_relation) + && window[2].token_type == TokenType::LParen + && window[3].token_type == TokenType::RParen + }) + .flat_map(|window| [window[2].span.start, window[3].span.start]) + .collect(); + if blanked.is_empty() { + return String::new(); + } + sql.chars() + .enumerate() + .map(|(offset, character)| { + if blanked.contains(&offset) { + ' ' + } else { + character + } + }) + .collect() +} diff --git a/crates/sqlquack-lint/src/lint/_helpers/terminal_shape.rs b/crates/sqlquack-lint/src/lint/_helpers/terminal_shape.rs new file mode 100644 index 000000000..c4a8b003c --- /dev/null +++ b/crates/sqlquack-lint/src/lint/_helpers/terminal_shape.rs @@ -0,0 +1,373 @@ +use sqlquack_syntax::tokens::models::{Span, Token, TokenType}; + +use crate::lint::_helpers::engine::{is_query_from, significant_before}; + +const CEREMONIAL_SELECT_LITERAL: &str = "1"; + +pub(super) fn final_cte_name_spans(tokens: &[Token], depths: &[usize]) -> Vec { + let roots = root_with_indices(tokens, depths); + let mut names: Vec<(usize, bool)> = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if token.token_type != TokenType::With { + continue; + } + let depth = depths[index]; + let first = names.len(); + let mut next = index + 1; + if tokens + .get(next) + .is_some_and(|token| token.text.eq_ignore_ascii_case("recursive")) + { + next += 1; + } + loop { + let name = next; + let Some(open) = cte_body_open(tokens, depths, name) else { + break; + }; + let Some(close) = (open + 1..tokens.len()).find(|&position| { + depths[position] == depth + 1 && tokens[position].token_type == TokenType::RParen + }) else { + break; + }; + names.push((name, false)); + if tokens + .get(close + 1) + .is_none_or(|token| token.token_type != TokenType::Comma) + { + break; + } + next = close + 2; + } + if roots.contains(&index) + && names.len() > first + && let Some((_, terminal)) = names.last_mut() + { + *terminal = true; + } + } + names + .into_iter() + .filter_map(|(index, terminal)| { + let is_final = tokens[index].text.eq_ignore_ascii_case("final"); + (terminal != is_final).then_some(tokens[index].span) + }) + .collect() +} + +pub(super) fn root_with_indices(tokens: &[Token], depths: &[usize]) -> Vec { + let significant: Vec = (0..tokens.len()) + .filter(|&index| { + !matches!( + tokens[index].token_type, + TokenType::Space + | TokenType::Break + | TokenType::LineComment + | TokenType::BlockComment + ) + }) + .collect(); + significant + .split(|&index| tokens[index].token_type == TokenType::Semicolon) + .filter_map(|statement| { + root_query_tokens(tokens, depths, statement) + .first() + .copied() + }) + .filter(|&index| tokens[index].token_type == TokenType::With) + .collect() +} + +fn root_query_tokens<'a>( + tokens: &[Token], + depths: &[usize], + mut significant: &'a [usize], +) -> &'a [usize] { + while significant + .last() + .is_some_and(|&index| tokens[index].token_type == TokenType::Semicolon) + { + significant = &significant[..significant.len() - 1]; + } + while let Some((&first, rest)) = significant.split_first() { + if tokens[first].token_type != TokenType::LParen + || rest + .last() + .is_none_or(|&index| tokens[index].token_type != TokenType::RParen) + || rest.iter().any(|&index| depths[index] <= depths[first]) + { + break; + } + significant = &rest[..rest.len() - 1]; + } + significant +} + +fn cte_body_open(tokens: &[Token], depths: &[usize], name: usize) -> Option { + let mut position = name + 1; + if tokens.get(position)?.token_type == TokenType::LParen { + position = (position + 1..tokens.len()).find(|&index| { + depths[index] == depths[name] + 1 && tokens[index].token_type == TokenType::RParen + })? + 1; + } + if tokens.get(position)?.token_type != TokenType::As { + return None; + } + position += 1; + if tokens.get(position)?.token_type == TokenType::Not { + position += 1; + } + if tokens + .get(position)? + .text + .eq_ignore_ascii_case("materialized") + { + position += 1; + } + (tokens.get(position)?.token_type == TokenType::LParen).then_some(position) +} + +pub(super) fn collect_terminal_shape_facts( + tokens: &[Token], + depths: &[usize], + significant: &[usize], + allows_ceremonial_select: bool, +) -> (Vec, Vec) { + let mut cte_only_bodies: Vec = Vec::new(); + let mut terminal_selects: Vec = Vec::new(); + let significant = root_query_tokens(tokens, depths, significant); + let Some(&first) = significant.first() else { + return (cte_only_bodies, terminal_selects); + }; + let root_depth = depths[first]; + let has_top_level_with = significant.iter().any(|&index| { + depths[index] == root_depth && tokens[index].text.eq_ignore_ascii_case("with") + }); + let terminal_set_operator = significant.iter().find(|&&index| { + depths[index] == root_depth + && matches!( + tokens[index].token_type, + TokenType::Union | TokenType::Intersect | TokenType::Except + ) + }); + if let Some(&operator) = terminal_set_operator { + cte_only_bodies.push(tokens[operator].span); + if has_top_level_with { + terminal_selects.push(tokens[operator].span); + } + return (cte_only_bodies, terminal_selects); + } + let Some(root_select_position) = significant.iter().position(|&index| { + depths[index] == root_depth && tokens[index].token_type == TokenType::Select + }) else { + return (cte_only_bodies, terminal_selects); + }; + let root_select = significant[root_select_position]; + let tail: Vec = significant[root_select_position + 1..] + .iter() + .copied() + .take_while(|&index| tokens[index].token_type != TokenType::Semicolon) + .filter(|&index| depths[index] == root_depth) + .collect(); + let has_terminal_logic = tail.iter().any(|&index| { + matches!( + tokens[index].token_type, + TokenType::Join + | TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Order + | TokenType::Limit + | TokenType::Offset + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + ) + }); + if !has_top_level_with || has_terminal_logic { + cte_only_bodies.push(tokens[root_select].span); + } + if !has_top_level_with { + return (cte_only_bodies, terminal_selects); + } + + let final_cte = (0..root_select_position) + .filter(|&position| { + let index = significant[position]; + depths[index] == root_depth + && tokens[index].token_type == TokenType::As + && significant + .get(position + 1) + .is_some_and(|&next| tokens[next].token_type == TokenType::LParen) + }) + .filter_map(|position| cte_name_before_as(tokens, significant, position)) + .next_back(); + let from_position = tail + .iter() + .position(|&index| is_terminal_from(tokens, significant, index)); + let terminal_is_plain = from_position.is_some_and(|position| { + let projection = &tail[..position]; + let relation = tail.get(position + 1); + let remainder = &tail[position.saturating_add(2)..]; + !projection.is_empty() + && plain_terminal_projection(tokens, projection) + && relation.is_some_and(|&index| { + final_cte + .as_ref() + .is_some_and(|name| tokens[index].text.eq_ignore_ascii_case(name)) + }) + && remainder.is_empty() + }); + let terminal_is_ceremonial = allows_ceremonial_select + && tail.len() == 1 + && tokens[tail[0]].text == CEREMONIAL_SELECT_LITERAL; + if !terminal_is_plain && !terminal_is_ceremonial { + terminal_selects.push(tokens[root_select].span); + } + (cte_only_bodies, terminal_selects) +} + +fn cte_name_before_as( + tokens: &[Token], + significant: &[usize], + as_position: usize, +) -> Option { + let previous_position = as_position.checked_sub(1)?; + let previous = significant[previous_position]; + if is_lint_identifier(&tokens[previous]) || tokens[previous].text.eq_ignore_ascii_case("final") + { + return Some(tokens[previous].text.to_ascii_lowercase()); + } + if tokens[previous].token_type != TokenType::RParen { + return None; + } + + let mut balance = 1_usize; + for position in (0..previous_position).rev() { + let index = significant[position]; + match tokens[index].token_type { + TokenType::RParen => balance += 1, + TokenType::LParen => { + balance = balance.saturating_sub(1); + if balance == 0 { + let name = position.checked_sub(1).map(|value| significant[value])?; + return (is_lint_identifier(&tokens[name]) + || tokens[name].text.eq_ignore_ascii_case("final")) + .then(|| tokens[name].text.to_ascii_lowercase()); + } + } + _ => {} + } + } + None +} + +fn plain_terminal_projection(tokens: &[Token], projection: &[usize]) -> bool { + projection.iter().all(|&index| { + is_terminal_projection_identifier(&tokens[index]) + || matches!( + tokens[index].token_type, + TokenType::Star | TokenType::Dot | TokenType::Comma | TokenType::As + ) + }) +} + +fn is_terminal_projection_identifier(token: &Token) -> bool { + is_lint_identifier(token) + || matches!( + token.token_type, + TokenType::Comment + | TokenType::Date + | TokenType::Key + | TokenType::Sequence + | TokenType::Time + ) + || (token.token_type == TokenType::Percent && token.text.eq_ignore_ascii_case("percent")) +} + +fn is_lint_identifier(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Var + ) +} + +fn is_terminal_from(tokens: &[Token], significant: &[usize], index: usize) -> bool { + if tokens[index].token_type != TokenType::From { + return false; + } + significant + .iter() + .position(|&candidate| candidate == index) + .and_then(|position| position.checked_sub(1)) + .is_none_or(|position| tokens[significant[position]].token_type != TokenType::Distinct) +} + +pub(super) fn first_row_selection_clause(tokens: &[Token], direct: &[usize]) -> Option { + let from = direct + .iter() + .copied() + .find(|&index| is_query_from(tokens, index)); + direct.iter().copied().find(|&index| { + matches!( + tokens[index].token_type, + TokenType::Limit | TokenType::Offset + ) && from.is_none_or(|from_index| index > from_index) + && significant_before(tokens, index).is_none_or(|previous| { + !matches!( + tokens[previous].token_type, + TokenType::Colon | TokenType::DColon | TokenType::Dot | TokenType::DotColon + ) + }) + }) +} + +/// Whether a row-selection clause is `LIMIT 0`, which selects no rows in any order. +pub(super) fn limits_to_no_rows(tokens: &[Token], index: usize) -> bool { + tokens[index].token_type == TokenType::Limit + && tokens.get(index + 1).is_some_and(|count| { + count.token_type == TokenType::Number && count.text.parse::() == Ok(0) + }) +} + +/// Whether the projection before `limit` holds only constants, so every row it could select is identical. +pub(super) fn projects_only_constants(tokens: &[Token], direct: &[usize], limit: usize) -> bool { + let Some(&first) = direct.first() else { + return false; + }; + let end: usize = direct + .iter() + .copied() + .find(|&index| is_query_from(tokens, index)) + .unwrap_or(limit); + (first..end).all(|index| match tokens[index].token_type { + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier => { + names_output_or_type(tokens, index) + } + TokenType::Star | TokenType::Select => false, + _ => true, + }) +} + +/// Whether the name at `index` is an alias or a type after `AS`, or a bare alias after a value. +fn names_output_or_type(tokens: &[Token], index: usize) -> bool { + let Some(previous) = index.checked_sub(1).map(|previous| &tokens[previous]) else { + return false; + }; + let followed_by_list_end: bool = tokens + .get(index + 1) + .is_none_or(|next| matches!(next.token_type, TokenType::Comma | TokenType::From)); + previous.token_type == TokenType::As + || (followed_by_list_end + && matches!( + previous.token_type, + TokenType::Number + | TokenType::String + | TokenType::Null + | TokenType::True + | TokenType::False + | TokenType::Parameter + | TokenType::RParen + )) +} diff --git a/crates/sqlquack-lint/src/lint/constants.rs b/crates/sqlquack-lint/src/lint/constants.rs new file mode 100644 index 000000000..8ffe73443 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/constants.rs @@ -0,0 +1,117 @@ +use sqlquack_syntax::tokens::models::TokenType; + +/// Dialect names SQLClean's engine accepted that SQLQuack does not parse yet; their lint is unavailable. +pub(crate) const UNPARSED_DIALECTS: [&str; 45] = [ + "", + "arrow-datafusion", + "arrow_datafusion", + "athena", + "bigquery", + "clickhouse", + "cockroach", + "cockroachdb", + "databricks", + "datafusion", + "doris", + "dremio", + "drill", + "druid", + "dune", + "exasol", + "fabric", + "generic", + "hana", + "hive", + "materialize", + "memsql", + "mssql", + "mysql", + "oracle", + "postgres", + "postgresql", + "presto", + "redshift", + "risingwave", + "sap_hana", + "saphana", + "singlestore", + "solr", + "spark", + "spark2", + "sqlite", + "sqlserver", + "starrocks", + "tableau", + "teradata", + "tidb", + "trino", + "tsql", + "vertica", +]; + +pub const LINT_API_VERSION: u32 = 1; +pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ + "ANTI", + "APPLY", + "ASOF", + "AT", + "BEFORE", + "CHANGES", + "CONNECT", + "CROSS", + "EXCEPT", + "FETCH", + "FINAL", + "FOR", + "FULL", + "GROUP", + "HAVING", + "INNER", + "INTERSECT", + "JOIN", + "LATERAL", + "LEFT", + "LIMIT", + "MATCH_RECOGNIZE", + "MINUS", + "NATURAL", + "OFFSET", + "ON", + "ORDER", + "OUTER", + "PASTE", + "PIVOT", + "POSITIONAL", + "QUALIFY", + "RETURNING", + "RIGHT", + "SAMPLE", + "SEMI", + "START", + "TABLESAMPLE", + "UNION", + "UNPIVOT", + "USING", + "WHERE", + "WINDOW", +]; +/// The parts of a `SQLCLEAN038` message that names its joined relation and CTE. +pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; +pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; +pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; +/// The `SQLCLEAN038` message for a join whose relation cannot be named. +pub(crate) const CROSS_JOIN_UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; +/// Stands for the finding's project-relative path until the caller that knows it fills it in. +pub(crate) const CROSS_JOIN_PATH_PLACEHOLDER: &str = ""; +pub(crate) const CROSS_JOIN_REASON_PLACEHOLDER: &str = ""; +/// Tokens that end a select item, so parentheses closed just before one cannot be regrouping an operator. +pub(crate) const SELECT_ITEM_END_TOKENS: [TokenType; 8] = [ + TokenType::Comma, + TokenType::From, + TokenType::As, + TokenType::Semicolon, + TokenType::RParen, + TokenType::Var, + TokenType::Identifier, + TokenType::QuotedIdentifier, +]; diff --git a/crates/sqlquack-lint/src/lint/main/batch_engine.rs b/crates/sqlquack-lint/src/lint/main/batch_engine.rs new file mode 100644 index 000000000..555b8163a --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/batch_engine.rs @@ -0,0 +1,19 @@ +use crate::lint::_helpers::engine::lint; +use crate::lint::models::{LintRequest, LintResponse}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +pub fn lint_batch_json(request_json: &str) -> Result { + let requests: Vec = + serde_json::from_str(request_json).map_err(|error| error.to_string())?; + if requests.is_empty() { + return Ok("[]".to_owned()); + } + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(requests.len().min(4)) + .stack_size(16 * 1024 * 1024) + .build() + .map_err(|error| error.to_string())?; + let results: Vec> = + pool.install(|| requests.into_par_iter().map(lint).collect()); + serde_json::to_string(&results).map_err(|error| error.to_string()) +} diff --git a/crates/sqlquack-lint/src/lint/main/catalogue.rs b/crates/sqlquack-lint/src/lint/main/catalogue.rs new file mode 100644 index 000000000..4ad99a79f --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/catalogue.rs @@ -0,0 +1,6 @@ +use crate::lint::_helpers::engine::rule_metadata; +use crate::lint::models::LintRuleMetadata; + +pub fn catalogue() -> Vec { + rule_metadata() +} diff --git a/crates/sqlquack-lint/src/lint/main/cross_join_subject.rs b/crates/sqlquack-lint/src/lint/main/cross_join_subject.rs new file mode 100644 index 000000000..f290e59a5 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/cross_join_subject.rs @@ -0,0 +1,15 @@ +//! The relation and CTE a `SQLCLEAN038` finding names, for relation-level Rule exceptions. + +use crate::lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; + +/// The `(relation, cte)` a cross join finding message names, `None` for the generic message. +#[must_use] +pub fn cross_join_subject(message: &str) -> Option<(String, Option)> { + let named: &str = message + .strip_prefix(CROSS_JOIN_PREFIX)? + .strip_suffix(CROSS_JOIN_SUFFIX)?; + match named.split_once(CROSS_JOIN_CTE_MARKER) { + Some((relation, cte)) => Some((relation.to_owned(), Some(cte.to_owned()))), + None => Some((named.to_owned(), None)), + } +} diff --git a/crates/sqlquack-lint/src/lint/main/default_rules.rs b/crates/sqlquack-lint/src/lint/main/default_rules.rs new file mode 100644 index 000000000..563e9f67f --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/default_rules.rs @@ -0,0 +1,6 @@ +//! The built-in Rules a family selector or an absent selection enables. + +/// The codes enabled when no Rule selection is given. +pub fn default_rules() -> Vec<&'static str> { + crate::lint::_helpers::engine::DEFAULT_RULES.to_vec() +} diff --git a/crates/sqlquack-lint/src/lint/main/engine.rs b/crates/sqlquack-lint/src/lint/main/engine.rs new file mode 100644 index 000000000..c31857ef5 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/engine.rs @@ -0,0 +1,5 @@ +use crate::lint::_helpers::engine::lint_json_impl; + +pub fn lint_json(request_json: &str) -> Result { + lint_json_impl(request_json) +} diff --git a/crates/sqlquack-lint/src/lint/main/fill_finding_path.rs b/crates/sqlquack-lint/src/lint/main/fill_finding_path.rs new file mode 100644 index 000000000..d18e35bee --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/fill_finding_path.rs @@ -0,0 +1,9 @@ +//! `fill_finding_path`: a SQL finding's help with its file's path written into the entry it prints. + +use crate::lint::constants::CROSS_JOIN_PATH_PLACEHOLDER; + +/// `remediation` with the path placeholder of a printed exception entry replaced by `path`. +#[must_use] +pub fn fill_finding_path(remediation: &str, path: &str) -> String { + remediation.replace(CROSS_JOIN_PATH_PLACEHOLDER, path) +} diff --git a/crates/sqlquack-lint/src/lint/main/lint.rs b/crates/sqlquack-lint/src/lint/main/lint.rs new file mode 100644 index 000000000..e10767564 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/lint.rs @@ -0,0 +1,8 @@ +//! Typed, in-process lint of one SQL text. + +use crate::lint::models::{LintRequest, LintResponse}; + +/// Lint `request.sql` with the selected built-in Rules and return findings in source order. +pub fn lint(request: LintRequest) -> Result { + crate::lint::_helpers::engine::lint(request) +} diff --git a/crates/sqlquack-lint/src/lint/main/mod.rs b/crates/sqlquack-lint/src/lint/main/mod.rs new file mode 100644 index 000000000..d0f6d2dee --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/mod.rs @@ -0,0 +1,9 @@ +pub mod batch_engine; +pub mod catalogue; +pub mod cross_join_subject; +pub mod default_rules; +pub mod engine; +pub mod fill_finding_path; +pub mod lint; +pub mod parse_sql; +pub mod validate_dialect; diff --git a/crates/sqlquack-lint/src/lint/main/parse_sql.rs b/crates/sqlquack-lint/src/lint/main/parse_sql.rs new file mode 100644 index 000000000..d98d96e9a --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/parse_sql.rs @@ -0,0 +1,11 @@ +//! Tokenize and parse SQL the way every lint request does. + +use crate::lint::models::ParsedSql; + +/// Parse `sql` under `dialect`, rejecting statements the parser kept as opaque text. +pub fn parse_sql(sql: &str, dialect: &str) -> Result { + let Some(parsed) = crate::lint::_helpers::parsing::dialect(dialect)? else { + return Err(format!("SQLQuack does not parse the {dialect} dialect yet")); + }; + crate::lint::_helpers::parsing::parse(sql, parsed, None) +} diff --git a/crates/sqlquack-lint/src/lint/main/validate_dialect.rs b/crates/sqlquack-lint/src/lint/main/validate_dialect.rs new file mode 100644 index 000000000..72c2ae9f6 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/main/validate_dialect.rs @@ -0,0 +1,9 @@ +//! Check a dialect name against the dialects SQL lint accepts. + +/// `Ok` when `name` names a known SQL dialect, such as `duckdb` or `snowflake`, parsed or not. +pub fn validate_dialect(name: &str) -> Result<(), String> { + match crate::lint::_helpers::parsing::dialect(name) { + Ok(_) => Ok(()), + Err(error) => Err(format!("unknown SQL dialect '{name}': {error}")), + } +} diff --git a/crates/sqlquack-lint/src/lint/mod.rs b/crates/sqlquack-lint/src/lint/mod.rs new file mode 100644 index 000000000..4a83e8c95 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/mod.rs @@ -0,0 +1,9 @@ +//! SQL lint Rules over SQLQuack's tokens and syntax trees. + +pub mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-lint/src/lint/models.rs b/crates/sqlquack-lint/src/lint/models.rs new file mode 100644 index 000000000..f5a719f13 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/models.rs @@ -0,0 +1,184 @@ +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; +use sqlquack_syntax::tokens::models::{Span, Token}; +use sqlquack_syntax::{Dialect, Expression}; + +/// SQL text tokenized and parsed under one dialect. +#[derive(Debug)] +pub struct ParsedSql { + pub dialect: Dialect, + pub tokens: Vec, + pub statements: Vec, +} + +#[derive(Debug, Default)] +pub(crate) struct QueryFacts { + pub null_comparisons: Vec, + pub implicit_cartesian_joins: Vec, + pub joins_without_condition: Vec, + pub unordered_limits: Vec, + pub redundant_distincts: Vec, + pub positional_set_stars: Vec, + pub unused_cte_names: Vec, + pub cross_joins: Vec, + pub cte_only_bodies: Vec, + pub terminal_selects: Vec, + pub nested_ctes: Vec, + pub recursive_ctes: Vec, + pub invalid_comment_attachments: Vec, + pub inline_query_relations: Vec, + pub additional: AdditionalQueryFacts, +} + +#[derive(Debug, Default)] +pub(crate) struct AdditionalQueryFacts { + pub bare_unions: Vec, + pub duplicate_table_aliases: Vec, + pub duplicate_output_aliases: Vec, + pub mixed_group_order_references: Vec, + pub redundant_else_nulls: Vec, + pub parenthesized_distincts: Vec, + pub constant_predicates: Vec, + pub consecutive_semicolons: Vec, + pub redundant_self_aliases: Vec, + pub count_one_literals: Vec, + pub unstable_row_numbers: Vec, + pub null_not_in_predicates: Vec, + pub set_arity_mismatches: Vec, + pub projected_stars: Vec, + pub unaliased_calculations: Vec, + pub unused_table_aliases: Vec, + pub null_rejected_left_joins: Vec, + pub implicit_inner_joins: Vec, + pub ambiguous_order_directions: Vec, + pub unqualified_multi_source_columns: Vec, + pub inconsistent_single_source_qualification: Vec, + pub unknown_relation_qualifiers: Vec, + pub simple_boolean_cases: Vec, + pub nested_else_cases: Vec, + pub unused_joined_relations: Vec, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LintRequest { + pub version: u32, + pub sql: String, + pub dialect: String, + #[serde(default)] + pub enabled_rules: Option>, + #[serde(default)] + pub ignored_rules: Vec, + #[serde(default)] + pub external_identifiers: Vec, + #[serde(default)] + pub dependency_identifiers: Vec, + #[serde(default)] + pub externally_referenced_ctes: Vec, + /// The harness the host wraps this body in; absent for an ordinary query. + #[serde(default)] + pub fixture_harness: Option, + #[serde(default)] + pub allows_dynamic_output_star: bool, + #[serde(default)] + pub max_literal_length: Option, + #[serde(default)] + pub max_ranking_order_by: Option, + #[serde(default)] + pub relation_keys: HashMap>>, + #[serde(default)] + pub relation_names: HashMap, + /// The SQL each macro call in the query generated, so column qualification is judged as the macro wrote it. + #[serde(default)] + pub sql_macro_outputs: Vec, + /// The configuration file remediation text names; SQLClean's own when absent. + #[serde(default)] + pub config_file: Option, + /// The advice `SQLCLEAN044` gives before its limit setting; SQLClean's own when absent. + #[serde(default)] + pub long_literal_guidance: Option, +} + +/// The generated wrapper around a test or macro body: its terminal SELECT is ceremonial, and these names are generated. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FixtureHarness { + /// Prefixes of the CTE names the harness generates, which are reachable even when unread. + #[serde(default)] + pub cte_prefixes: Vec, + /// Prefixes of the CTEs standing in for the body's inputs, which may read the empty relation with `SELECT *`. + #[serde(default)] + pub input_cte_prefixes: Vec, + /// The table function an input CTE reads, as `SELECT * FROM ()`, when it has no rows. + #[serde(default)] + pub empty_relation: Option, +} + +/// The character range one macro call's output fills, and the relations its arguments pass, which a plain parameter slot read expands to qualify with. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub struct SqlMacroOutput { + pub start: usize, + pub end: usize, + #[serde(default)] + pub relations: Vec, +} + +#[derive(Debug)] +pub(crate) struct AdditionalFactOptions<'a> { + pub external_identifiers: &'a HashSet, + pub dependency_identifiers: &'a HashSet, + pub sql_macro_outputs: &'a [SqlMacroOutput], + pub allows_ceremonial_select: bool, + pub allows_dynamic_output_star: bool, + pub fixture_harness: Option<&'a FixtureHarness>, +} + +#[derive(Debug, Serialize)] +pub struct LintResponse { + pub version: u32, + pub diagnostics: Vec, + /// Why no Rule ran, for a dialect SQLQuack does not parse yet; absent when the SQL was linted. + #[serde(skip_serializing_if = "Option::is_none")] + pub unavailable: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LintDiagnostic { + pub code: &'static str, + pub message: Cow<'static, str>, + pub remediation: Cow<'static, str>, + pub start: usize, + pub end: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub fix: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fix_unavailable_reason: Option<&'static str>, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LintEdit { + pub start: usize, + pub end: usize, + pub replacement: String, +} + +#[derive(Debug, Clone, Copy)] +pub struct LintRuleMetadata { + pub code: &'static str, + pub message: &'static str, + pub remediation: &'static str, +} + +/// One lint request's explicit cross joins, with what is known about the relations they read. +#[derive(Debug)] +pub(crate) struct CrossJoinRequest<'a> { + pub(crate) statements: &'a [Expression], + pub(crate) tokens: &'a [Token], + pub(crate) cross_spans: &'a [Span], + /// Declared keys by dependency sentinel; the empty key proves at most one row. + pub(crate) relation_keys: &'a HashMap>>, + /// The relation name each dependency sentinel stands for. + pub(crate) relation_names: &'a HashMap, +} diff --git a/crates/sqlquack-lint/src/lint/tests.rs b/crates/sqlquack-lint/src/lint/tests.rs new file mode 100644 index 000000000..308f1193d --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests.rs @@ -0,0 +1,18 @@ +#[path = "tests/test_batch_lint.rs"] +mod batch_lint; +#[path = "tests/test_cross_joins.rs"] +mod cross_joins; +#[path = "tests/test_empty_fixture_lint.rs"] +mod empty_fixture_lint; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_lint_exemptions.rs"] +mod lint_exemptions; +#[path = "tests/test_long_chains.rs"] +mod long_chains; +#[path = "tests/test_sql_lint.rs"] +mod sql_lint; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_unused_table_alias.rs"] +mod unused_table_alias; diff --git a/crates/sqlquack-lint/src/lint/tests/helpers.rs b/crates/sqlquack-lint/src/lint/tests/helpers.rs new file mode 100644 index 000000000..2174fcb88 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/helpers.rs @@ -0,0 +1,104 @@ +use serde_json::{Map, Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::lint::tests::test_types::CrossJoinTestCase; + +/// The dependency sentinel the cross join cases name `order_policy`. +pub(crate) const POLICY_SENTINEL: &str = "__sqlclean_interpolation_7__"; + +pub(crate) fn nested_function_sql(depth: usize) -> String { + format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) +} + +pub(crate) fn nested_from_sql(depth: usize) -> String { + format!( + "SELECT * FROM {}orders{}", + "(".repeat(depth), + ")".repeat(depth) + ) +} + +pub(crate) fn diagnostics(sql: &str) -> Result, String> { + let response = lint_json( + &json!({ + "version": 1, + "sql": sql, + "dialect": "snowflake" + }) + .to_string(), + )?; + diagnostic_values(&response) +} + +pub(crate) fn diagnostics_for_rules(sql: &str, rules: &[&str]) -> Result, String> { + diagnostics_for_dialect(sql, "snowflake", rules) +} + +pub(crate) fn diagnostics_for_dialect( + sql: &str, + dialect: &str, + rules: &[&str], +) -> Result, String> { + let request = json!({ + "version": 1, + "sql": sql, + "dialect": dialect, + "enabled_rules": rules, + }); + let response = lint_json(&request.to_string())?; + diagnostic_values(&response) +} + +fn diagnostic_values(response: &str) -> Result, String> { + let payload: Value = serde_json::from_str(response).map_err(|error| error.to_string())?; + payload["diagnostics"] + .as_array() + .cloned() + .ok_or_else(|| "diagnostics should be an array".to_string()) +} + +/// The `SQLCLEAN038` messages for one cross join case. +pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result, String> { + cross_join_texts(test_case, "message") +} + +/// The `SQLCLEAN038` remediations for one cross join case. +pub(crate) fn cross_join_remediations( + test_case: &CrossJoinTestCase, +) -> Result, String> { + cross_join_texts(test_case, "remediation") +} + +fn cross_join_texts(test_case: &CrossJoinTestCase, field: &str) -> Result, String> { + let keys: Map = test_case + .one_row_relations + .iter() + .map(|name| ((*name).to_owned(), json!([[]]))) + .collect(); + let request = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "enabled_rules": ["SQLCLEAN038"], + "relation_keys": keys, + "relation_names": { POLICY_SENTINEL: "order_policy" }, + }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default() + .iter() + .map(|diagnostic| diagnostic[field].as_str().unwrap_or_default().to_owned()) + .collect()) +} + +/// A host fixture harness like a test runner's: generated CTE prefixes, input stand-ins and an empty relation. +pub(crate) fn fixture_harness() -> Value { + json!({ + "cte_prefixes": ["__expected__", "__ref__", "__source__", "__macro_actual__", "__macro_expected__"], + "input_cte_prefixes": ["__ref__", "__source__", "__seed__"], + "empty_relation": "__empty_fixture", + }) +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_batch_lint.rs b/crates/sqlquack-lint/src/lint/tests/test_batch_lint.rs new file mode 100644 index 000000000..9880125bf --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_batch_lint.rs @@ -0,0 +1,43 @@ +use crate::lint::main::batch_engine::lint_batch_json; +use crate::lint::main::engine::lint_json; +use crate::lint::tests::test_types::BatchLintTestCase; +use serde_json::{Value, json}; + +#[test] +fn given_mixed_lint_inputs_when_batching_then_preserves_order_errors_and_findings() { + let test_cases = [BatchLintTestCase { + description: "mixed valid and invalid SQL retains standalone evidence", + queries: &[ + "SELECT 1", + "SELECT FROM", + "SELECT order_id FROM orders WHERE order_id = NULL", + ], + expected_responses: 3, + }]; + for test_case in test_cases { + let requests: Vec = test_case + .queries + .iter() + .map(|sql| json!({"version": 1, "sql": sql, "dialect": "duckdb"})) + .collect(); + let expected: Vec = requests + .iter() + .map(|request| { + serde_json::to_value(lint_json(&request.to_string()).map(|response| { + serde_json::from_str::(&response).expect("valid response") + })) + .expect("serializable result") + }) + .collect(); + let response = lint_batch_json(&serde_json::to_string(&requests).expect("valid requests")) + .expect("valid batch"); + let actual: Vec = serde_json::from_str(&response).expect("valid response"); + assert_eq!( + actual.len(), + test_case.expected_responses, + "{}", + test_case.description + ); + assert_eq!(actual, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_cross_joins.rs b/crates/sqlquack-lint/src/lint/tests/test_cross_joins.rs new file mode 100644 index 000000000..680037aa4 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_cross_joins.rs @@ -0,0 +1,284 @@ +use crate::lint::main::cross_join_subject::cross_join_subject; +use crate::lint::tests::helpers::{POLICY_SENTINEL, cross_join_messages, cross_join_remediations}; +use crate::lint::tests::test_types::{CrossJoinSubjectTestCase, CrossJoinTestCase}; + +const GENERIC_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product"; +const UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; + +#[test] +fn given_cross_joins_when_one_side_is_provably_one_row_then_only_multiplying_joins_are_reported() +-> Result<(), String> { + let test_cases = [ + CrossJoinTestCase { + description: "two unknown relations multiply rows", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the right", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqlclean_interpolation_7__ AS p", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "a seed with several rows names its relation", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqlclean_interpolation_7__ AS p", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the left of the first join", + sql: "SELECT o.order_id FROM __sqlclean_interpolation_7__ AS p CROSS JOIN orders AS o", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate without GROUP BY carried through a filtering CTE", + sql: "WITH totals AS (SELECT COUNT(*) AS order_count FROM orders), \ + kept AS (SELECT order_count FROM totals WHERE order_count > 0) \ + SELECT o.order_id, k.order_count FROM orders AS o CROSS JOIN kept AS k", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate with GROUP BY can return many rows", + sql: "WITH totals AS (SELECT customer_id, COUNT(*) AS order_count FROM orders GROUP BY customer_id) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a windowed count is not an aggregate", + sql: "WITH totals AS (SELECT COUNT(*) OVER () AS order_count FROM orders) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "LIMIT 1", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "LIMIT 2 can return two rows", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 2) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `latest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "one-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1, 'EUR')) AS v(version, currency)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "two-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1), (2)) AS v(version)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `settings` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 without a partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 with a partition keeps one row per partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a join inside the carrying CTE stops the proof", + sql: "WITH totals AS (SELECT COUNT(*) AS n FROM orders), \ + joined AS (SELECT t.n FROM totals AS t INNER JOIN customers AS c ON c.n = t.n) \ + SELECT o.order_id FROM orders AS o CROSS JOIN joined AS j", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `joined` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "UNNEST in a carrying CTE stops the proof", + sql: "WITH totals AS (SELECT LIST(order_id) AS ids FROM orders), \ + expanded AS (SELECT UNNEST(ids) AS order_id FROM totals) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN expanded AS e", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `expanded` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a set operation stops the proof", + sql: "WITH both_totals AS (SELECT COUNT(*) AS n FROM orders UNION ALL SELECT COUNT(*) AS n FROM customers) \ + SELECT o.order_id FROM orders AS o CROSS JOIN both_totals AS b", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `both_totals` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "the finding names the CTE that contains the join", + sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ + SELECT order_id FROM checks", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a later join still multiplies the rows before it", + sql: "SELECT o.order_id FROM __sqlclean_interpolation_7__ AS p CROSS JOIN orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "an unnamed relation says only a file-wide exception covers it", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[UNNAMED_MESSAGE], + }, + CrossJoinTestCase { + description: "a named window is never treated as unpartitioned", + sql: "WITH newest AS (SELECT order_id FROM orders WINDOW w AS (PARTITION BY customer_id ORDER BY order_id DESC) QUALIFY ROW_NUMBER() OVER w = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a join under a nested WITH names the top-level CTE", + sql: "WITH outer_checks AS (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) \ + SELECT order_id FROM outer_checks", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `outer_checks` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a WITH inside a subquery of the final select names no CTE", + sql: "SELECT s.order_id FROM (WITH inner_checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) SELECT order_id FROM inner_checks) AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a WITH owned by a set operation names its CTE", + sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ + SELECT order_id FROM checks UNION ALL SELECT order_id FROM orders", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + ]; + + for test_case in test_cases { + let messages: Vec = cross_join_messages(&test_case)?; + + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_unnamed_cross_join_when_linting_then_help_prints_a_file_wide_exception() +-> Result<(), String> { + let test_cases = [CrossJoinTestCase { + description: "CROSS JOIN UNNEST", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "[[rules.rule_exceptions]]\n rule = \"SQLCLEAN038\"\n path = \"\"\n reason = \"\"", + ], + }]; + + for test_case in test_cases { + let remediations: Vec = cross_join_remediations(&test_case)?; + + assert_eq!(remediations.len(), 1, "{}", test_case.description); + assert!( + remediations[0].contains(test_case.expected_messages[0]), + "{}: {remediations:?}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_cross_join_messages_when_reading_their_subject_then_relation_and_cte_are_named() { + let test_cases = [ + CrossJoinSubjectTestCase { + description: "a top-level join", + message: "CROSS JOIN with `order_policy` creates an explicit cartesian product", + expected_subject: Some(("order_policy", None)), + }, + CrossJoinSubjectTestCase { + description: "a join inside a CTE", + message: "CROSS JOIN with `order_policy` in CTE `order_checks` creates an explicit cartesian product", + expected_subject: Some(("order_policy", Some("order_checks"))), + }, + CrossJoinSubjectTestCase { + description: "the generic message names nothing", + message: GENERIC_MESSAGE, + expected_subject: None, + }, + ]; + + for test_case in test_cases { + let subject = cross_join_subject(test_case.message); + + assert_eq!( + subject + .as_ref() + .map(|(relation, cte)| (relation.as_str(), cte.as_deref())), + test_case.expected_subject, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_empty_fixture_lint.rs b/crates/sqlquack-lint/src/lint/tests/test_empty_fixture_lint.rs new file mode 100644 index 000000000..8cab7bbb0 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_empty_fixture_lint.rs @@ -0,0 +1,80 @@ +use serde_json::{Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::lint::tests::helpers::fixture_harness; +use crate::lint::tests::test_types; + +#[test] +fn given_empty_fixture_context_when_linting_projected_star_then_only_canonical_inputs_pass() +-> Result<(), String> { + let test_cases = [ + test_types::EmptyFixtureLintTestCase { + description: "canonical ref fixture", + sql: "WITH __ref__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", + dialect: "snowflake", + in_fixture_harness: true, + expected_diagnostic_count: 0, + }, + test_types::EmptyFixtureLintTestCase { + description: "canonical source fixture", + sql: "WITH __source__web__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", + dialect: "snowflake", + in_fixture_harness: true, + expected_diagnostic_count: 0, + }, + test_types::EmptyFixtureLintTestCase { + description: "ordinary SQL has no intrinsic exemption", + sql: "SELECT * FROM __empty_fixture()", + dialect: "duckdb", + in_fixture_harness: false, + expected_diagnostic_count: 1, + }, + test_types::EmptyFixtureLintTestCase { + description: "expected fixture remains controlled", + sql: "WITH __expected__orders AS (SELECT * FROM __empty_fixture()) SELECT 1", + dialect: "snowflake", + in_fixture_harness: true, + expected_diagnostic_count: 1, + }, + test_types::EmptyFixtureLintTestCase { + description: "additional projection is not canonical", + sql: "WITH __ref__orders AS (SELECT *, 1 AS extra FROM __empty_fixture()) SELECT 1", + dialect: "snowflake", + in_fixture_harness: true, + expected_diagnostic_count: 1, + }, + ]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "enabled_rules": ["SQLCLEAN021"], + "fixture_harness": test_case.in_fixture_harness.then(fixture_harness) + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + assert_eq!( + payload["diagnostics"].as_array().map_or(0, Vec::len), + test_case.expected_diagnostic_count, + "{}", + test_case.description + ); + } + let outside_harness = lint_json( + &json!({ + "version": 1, + "sql": "SELECT * FROM __empty_fixture()", + "dialect": "snowflake", + "enabled_rules": ["SQLCLEAN021"], + }) + .to_string(), + ); + assert!( + outside_harness.is_err(), + "Snowflake needs TABLE( ) around a table function outside the fixture harness" + ); + Ok(()) +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_lint_exemptions.rs b/crates/sqlquack-lint/src/lint/tests/test_lint_exemptions.rs new file mode 100644 index 000000000..2eda420f3 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_lint_exemptions.rs @@ -0,0 +1,173 @@ +use serde_json::{Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::lint::tests::helpers; +use crate::lint::tests::test_types::{ + DynamicOutputStarLintTestCase, LintTestCase, SqlMacroOutputLintTestCase, +}; + +#[test] +fn given_sql_macro_output_when_linting_qualification_then_each_column_counts_as_written() +-> Result<(), String> { + let test_cases = [ + SqlMacroOutputLintTestCase { + description: "a bare column from an expression macro beside a qualified column fails", + sql: "SELECT o.order_id, status AS s FROM orders AS o", + output: "status", + relations: &[], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "reads through a passed relation pass beside a bare caller column", + sql: "SELECT order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], + expected_count: 0, + }, + SqlMacroOutputLintTestCase { + description: "reads through a passed relation pass beside a hand-qualified column", + sql: "SELECT o.order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], + expected_count: 0, + }, + SqlMacroOutputLintTestCase { + description: "the caller's own mix beside reads through a passed relation fails", + sql: "SELECT o.order_id, status, o.amount AS a FROM orders AS o", + output: "o.amount", + relations: &["o"], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "an aliased read counts as qualified beside a bare caller column", + sql: "SELECT order_id, s.amount AS a FROM orders AS s", + output: "s.amount", + relations: &["orders"], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "a query scope the macro generated whole is judged in the macro file", + sql: "WITH c AS (SELECT a, w.b FROM w) SELECT * FROM c", + output: "SELECT a, w.b FROM w", + relations: &[], + expected_count: 0, + }, + ]; + for test_case in test_cases { + let start: usize = test_case + .sql + .find(test_case.output) + .ok_or("the output should be in the SQL")?; + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQLCLEAN028"], + "sql_macro_outputs": [{ + "start": start, + "end": start + test_case.output.len(), + "relations": test_case.relations, + }], + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() +-> Result<(), String> { + let test_cases = [DynamicOutputStarLintTestCase { + description: "unrelated CTE star remains controlled", + sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ +final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ +unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", + expected_count: 1, + expected_prefix: "unrelated AS (SELECT ", + }]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "snowflake", + "enabled_rules": ["SQLCLEAN021"], + "allows_dynamic_output_star": true, + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; + assert!( + test_case.sql[..start].ends_with(test_case.expected_prefix), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_constant_projection_when_linting_row_selection_then_only_column_dependent_rows_fail() +-> Result<(), String> { + let test_cases = [ + LintTestCase { + description: "constant-only projection selects identical rows in any order", + sql: "SELECT CAST(100 AS DOUBLE) AS measured_value, CAST(NULL AS INTEGER), 0 samples FROM items LIMIT 1", + expected_codes: &[], + expected_anchors: &[], + }, + LintTestCase { + description: "a column reference inside a cast still makes row selection nondeterministic", + sql: "SELECT CAST(id AS DOUBLE) AS measured_value FROM items LIMIT 1", + expected_codes: &["SQLCLEAN004"], + expected_anchors: &[("SQLCLEAN004", "LIMIT")], + }, + LintTestCase { + description: "a function call in a constant-looking projection still needs ordering", + sql: "SELECT now() AS observed_at FROM items OFFSET 1", + expected_codes: &["SQLCLEAN004"], + expected_anchors: &[("SQLCLEAN004", "OFFSET")], + }, + ]; + for test_case in &test_cases { + let diagnostics = helpers::diagnostics(test_case.sql)?; + let found: Vec<(&str, &str)> = diagnostics + .iter() + .filter_map(|item| { + let start = item["start"].as_u64()? as usize; + let end = item["end"].as_u64()? as usize; + Some((item["code"].as_str()?, &test_case.sql[start..end])) + }) + .collect(); + assert_eq!( + found, test_case.expected_anchors, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_long_chains.rs b/crates/sqlquack-lint/src/lint/tests/test_long_chains.rs new file mode 100644 index 000000000..f1ba9b0ed --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_long_chains.rs @@ -0,0 +1,75 @@ +use std::thread; + +use crate::lint::tests::helpers::diagnostics_for_dialect; +use crate::lint::tests::test_types::LongChainTestCase; + +const STACK_BYTES: usize = 2 * 1024 * 1024; +const TERMS: usize = 50_000; + +#[test] +fn given_long_operator_chains_under_a_cross_join_when_linting_on_a_small_stack_then_lint_finishes() +{ + let test_cases = [ + LongChainTestCase { + description: "Snowflake: a projection chain beside a CROSS JOIN", + dialect: "snowflake", + template: "SELECT {chain} FROM a CROSS JOIN b", + term: "1", + separator: " + ", + terms: TERMS, + expected_lints: true, + }, + LongChainTestCase { + description: "DuckDB: a projection chain beside a CROSS JOIN", + dialect: "duckdb", + template: "SELECT {chain} FROM a CROSS JOIN b", + term: "1", + separator: " + ", + terms: TERMS, + expected_lints: true, + }, + LongChainTestCase { + description: "Snowflake: a chain in a cross-joined CTE's projection", + dialect: "snowflake", + template: "WITH c AS (SELECT {chain} AS x FROM t) SELECT c.x FROM a CROSS JOIN c", + term: "1", + separator: " + ", + terms: TERMS, + expected_lints: true, + }, + LongChainTestCase { + description: "Snowflake: a chain in the final projection after a CTE", + dialect: "snowflake", + template: "WITH c AS (SELECT t.x FROM t), d AS (SELECT c.x FROM c) SELECT {chain} FROM d CROSS JOIN e", + term: "1", + separator: " + ", + terms: TERMS, + expected_lints: true, + }, + LongChainTestCase { + description: "Snowflake: a set-operation chain is refused by the tree depth guard, not a crash", + dialect: "snowflake", + template: "WITH c AS ({chain}), r AS (SELECT c.a FROM c QUALIFY ROW_NUMBER() OVER (PARTITION BY c.a ORDER BY c.a) = 1) SELECT r.a FROM r", + term: "SELECT t.a FROM t", + separator: " INTERSECT ", + terms: TERMS, + expected_lints: false, + }, + ]; + for test_case in &test_cases { + let chain: String = vec![test_case.term; test_case.terms].join(test_case.separator); + let sql: String = test_case.template.replace("{chain}", &chain); + let dialect: &'static str = test_case.dialect; + let linted: bool = thread::Builder::new() + .stack_size(STACK_BYTES) + .spawn(move || diagnostics_for_dialect(&sql, dialect, &["SQLCLEAN"]).is_ok()) + .expect("the lint thread should start") + .join() + .expect("lint should not panic"); + assert_eq!( + linted, test_case.expected_lints, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs b/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs new file mode 100644 index 000000000..12dfb0c3d --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs @@ -0,0 +1,1923 @@ +use serde_json::{Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::lint::tests::helpers::fixture_harness; +use crate::lint::tests::{helpers, test_types}; + +#[test] +fn given_production_function_depth_when_linting_then_bounded_parser_accepts_it() { + let test_cases = [test_types::FunctionDepthTestCase { + description: "production depth remains supported", + depth: 65, + expected_diagnostic_count: 0, + }]; + for test_case in test_cases { + let sql = helpers::nested_function_sql(test_case.depth); + let diagnostics = helpers::diagnostics(&sql).unwrap_or_else(|error| { + panic!("{}: {error}", test_case.description); + }); + assert_eq!( + diagnostics.len(), + test_case.expected_diagnostic_count, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_excessive_function_depth_when_linting_then_complexity_guard_rejects_it() { + let test_cases = [test_types::FunctionDepthFailureTestCase { + description: "excessive depth remains bounded", + depth: 129, + expected_code: "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED", + expected_limit: "configured limit 128", + }]; + for test_case in test_cases { + let sql = helpers::nested_function_sql(test_case.depth); + let error = + helpers::diagnostics(&sql).expect_err("excessive nested calls should remain bounded"); + assert!( + error.contains(test_case.expected_code), + "{}", + test_case.description + ); + assert!( + error.contains(test_case.expected_limit), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deeply_nested_from_parentheses_when_linting_then_each_parser_refuses_cleanly() { + let test_cases = [ + test_types::NestingRefusalTestCase { + description: "Snowflake refuses past its nesting limit", + dialect: "snowflake", + depth: 1500, + expected_code: "E_GUARD_NESTING_DEPTH_EXCEEDED", + }, + test_types::NestingRefusalTestCase { + description: "DuckDB refuses past its nesting limit", + dialect: "duckdb", + depth: 1500, + expected_code: "E_GUARD_NESTING_DEPTH_EXCEEDED", + }, + ]; + for test_case in test_cases { + let sql = helpers::nested_from_sql(test_case.depth); + let error = helpers::diagnostics_for_dialect(&sql, test_case.dialect, &["SQLCLEAN001"]) + .expect_err("deeply nested FROM parentheses should be refused"); + assert!( + error.contains(test_case.expected_code), + "{}: {error}", + test_case.description + ); + } +} + +#[test] +fn given_sql_cases_when_linting_then_diagnostics_match() -> Result<(), String> { + let test_cases = [ + test_types::LintTestCase { + description: "NULL comparison", + sql: "SELECT value FROM a WHERE value = NULL", + expected_codes: &["SQLCLEAN001"], + expected_anchors: &[("SQLCLEAN001", "=")], + }, + test_types::LintTestCase { + description: "wrapped NULL comparison", + sql: "SELECT value FROM a WHERE (NULL) = value", + expected_codes: &["SQLCLEAN001"], + expected_anchors: &[("SQLCLEAN001", "=")], + }, + test_types::LintTestCase { + description: "implicit cartesian join", + sql: "SELECT a.id FROM a, b", + expected_codes: &["SQLCLEAN002"], + expected_anchors: &[("SQLCLEAN002", ",")], + }, + test_types::LintTestCase { + description: "unconditioned join", + sql: "SELECT a.id FROM a JOIN b", + expected_codes: &["SQLCLEAN003"], + expected_anchors: &[("SQLCLEAN003", "JOIN")], + }, + test_types::LintTestCase { + description: "parenthesized join condition is a condition", + sql: "SELECT a.id FROM a JOIN b ON (a.id = b.id OR a.status = 'new')", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "parenthesized constant join condition is no condition", + sql: "SELECT a.id FROM a JOIN b ON (1 = 1)", + expected_codes: &["SQLCLEAN003"], + expected_anchors: &[("SQLCLEAN003", "JOIN")], + }, + test_types::LintTestCase { + description: "unordered limit", + sql: "SELECT id FROM items LIMIT 1", + expected_codes: &["SQLCLEAN004"], + expected_anchors: &[("SQLCLEAN004", "LIMIT")], + }, + test_types::LintTestCase { + description: "limit zero selects no rows in any order", + sql: "SELECT id FROM items LIMIT 0", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "limit zero with an offset still selects no rows", + sql: "SELECT id FROM items LIMIT 0 OFFSET 5", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "semi-structured limit and offset keys are not row selection clauses", + sql: "SELECT payload:Limit::VARCHAR AS order_limit, payload:Offset::NUMBER AS order_offset FROM orders", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "semi-structured limit key does not hide a later row limit clause", + sql: "SELECT payload:Limit::VARCHAR AS order_limit FROM orders LIMIT 1", + expected_codes: &["SQLCLEAN004"], + expected_anchors: &[("SQLCLEAN004", "LIMIT")], + }, + test_types::LintTestCase { + description: "semi-structured limit key in a filter is not a row limit clause", + sql: "SELECT order_id FROM orders WHERE payload:Limit::NUMBER > 3", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "unused CTE", + sql: "WITH unused AS (SELECT 1) SELECT 1", + expected_codes: &["SQLCLEAN005"], + expected_anchors: &[("SQLCLEAN005", "unused")], + }, + test_types::LintTestCase { + description: "unreachable CTE chain beside a CTE both it and the main query read", + sql: "WITH used AS (SELECT 1 AS id), first AS (SELECT * FROM used), second AS (SELECT * FROM first) SELECT id FROM used", + expected_codes: &["SQLCLEAN005", "SQLCLEAN005"], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "unused CTE on set operation", + sql: "WITH unused AS (SELECT 1 AS id) SELECT id FROM left_table UNION ALL SELECT id FROM right_table", + expected_codes: &["SQLCLEAN005"], + expected_anchors: &[("SQLCLEAN005", "unused")], + }, + test_types::LintTestCase { + description: "redundant distinct", + sql: "SELECT DISTINCT id FROM items GROUP BY id", + expected_codes: &["SQLCLEAN006"], + expected_anchors: &[("SQLCLEAN006", "DISTINCT")], + }, + test_types::LintTestCase { + description: "aggregate distinct is not a select modifier", + sql: "SELECT category, COUNT(DISTINCT id) FROM items GROUP BY category", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "positional set star", + sql: "SELECT * FROM a UNION ALL SELECT * FROM b", + expected_codes: &["SQLCLEAN007"], + expected_anchors: &[("SQLCLEAN007", "UNION")], + }, + test_types::LintTestCase { + description: "parenthesized positional set star", + sql: "(SELECT * FROM a) UNION ALL (SELECT id FROM b)", + expected_codes: &["SQLCLEAN007"], + expected_anchors: &[("SQLCLEAN007", "UNION")], + }, + test_types::LintTestCase { + description: "nested set wildcard does not contaminate outer set operation", + sql: "SELECT 'actual' WHERE EXISTS (SELECT id FROM actual EXCEPT SELECT * FROM expected) UNION ALL SELECT 'expected'", + expected_codes: &["SQLCLEAN007"], + expected_anchors: &[("SQLCLEAN007", "EXCEPT")], + }, + test_types::LintTestCase { + description: "multiple statements", + sql: "SELECT value FROM first LIMIT 1; SELECT value FROM second OFFSET 1", + expected_codes: &["SQLCLEAN004", "SQLCLEAN004"], + expected_anchors: &[("SQLCLEAN004", "LIMIT"), ("SQLCLEAN004", "OFFSET")], + }, + test_types::LintTestCase { + description: "nested query anchors", + sql: "SELECT * FROM (SELECT id FROM inner_items ORDER BY id LIMIT 1) safe JOIN outer_items ON TRUE OFFSET 2", + expected_codes: &["SQLCLEAN039", "SQLCLEAN003", "SQLCLEAN004"], + expected_anchors: &[ + ("SQLCLEAN039", "SELECT"), + ("SQLCLEAN003", "JOIN"), + ("SQLCLEAN004", "OFFSET"), + ], + }, + test_types::LintTestCase { + description: "safe NULL predicate", + sql: "SELECT value FROM a WHERE value IS NULL", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "NULL inside function", + sql: "SELECT value FROM a WHERE COALESCE(value, NULL) = 1", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "NULL assignment", + sql: "UPDATE items SET value = NULL", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "cross join", + sql: "SELECT a.id FROM a CROSS JOIN b", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "keyed join", + sql: "SELECT a.id FROM a JOIN b ON a.id = b.id", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "ordered limit", + sql: "SELECT id FROM items ORDER BY id LIMIT 1", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "used CTE", + sql: "WITH used AS (SELECT 1 AS id) SELECT id FROM used", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "group without distinct", + sql: "SELECT id FROM items GROUP BY id", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "nonredundant distinct", + sql: "SELECT DISTINCT a FROM items GROUP BY a, b", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "within group", + sql: "SELECT DISTINCT LISTAGG(x, ',') WITHIN GROUP (ORDER BY x) FROM items", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "explicit positional set", + sql: "SELECT id FROM a UNION ALL SELECT id FROM b", + expected_codes: &[], + expected_anchors: &[], + }, + test_types::LintTestCase { + description: "nested star outside set arms", + sql: "SELECT id, name FROM (SELECT * FROM base) b UNION ALL SELECT id, name FROM other", + expected_codes: &["SQLCLEAN039"], + expected_anchors: &[("SQLCLEAN039", "SELECT")], + }, + test_types::LintTestCase { + description: "nested star inside wrapped set arm", + sql: "(SELECT id FROM (SELECT * FROM base) b) UNION ALL (SELECT id FROM other)", + expected_codes: &["SQLCLEAN039"], + expected_anchors: &[("SQLCLEAN039", "SELECT")], + }, + ]; + + for test_case in &test_cases { + let diagnostics = helpers::diagnostics(test_case.sql)?; + let codes: Vec<&str> = diagnostics + .iter() + .filter_map(|item| item["code"].as_str()) + .collect(); + let anchors: Vec<(&str, &str)> = diagnostics + .iter() + .filter_map(|item| { + let code = item["code"].as_str()?; + let start = item["start"].as_u64()? as usize; + let end = item["end"].as_u64()? as usize; + Some((code, &test_case.sql[start..end])) + }) + .collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + for expected_anchor in test_case.expected_anchors { + assert!( + anchors.contains(expected_anchor), + "{}", + test_case.description + ); + } + } + Ok(()) +} + +#[test] +fn given_plain_sql_rules_when_linting_then_project_context_is_not_required() -> Result<(), String> { + let test_cases = [ + test_types::PlainSqlLintTestCase { + description: "trailing comment", + sql: "SELECT id -- why\nFROM items", + rule: "SQLCLEAN033", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "detached comment", + sql: "SELECT id\n-- why\n\nFROM items", + rule: "SQLCLEAN033", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "attached predicate comment", + sql: "SELECT id FROM items WHERE active\n-- retain settled rows\nAND settled", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached CTE comment", + sql: "WITH\n-- normalized rows\nnormalized AS (SELECT id FROM items)\nSELECT id FROM normalized", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached join comment", + sql: "SELECT left_rows.id FROM left_rows\n-- match stable identities\nJOIN right_rows ON left_rows.id = right_rows.id", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached filter comment", + sql: "SELECT id FROM items\n-- retain active rows\nWHERE active", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached projection comment", + sql: "SELECT\n-- stable identity\nid\nFROM items", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached CASE branch comment", + sql: "SELECT CASE\n-- settled outcome\nWHEN settled THEN 1 ELSE 0 END FROM items", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "attached ordering comment with Unicode and CRLF", + sql: "SELECT id FROM items\r\n-- deterministic café order\r\nORDER BY id", + rule: "SQLCLEAN033", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "block comment", + sql: "SELECT id\n/* selected identity */\nFROM items", + rule: "SQLCLEAN033", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "body without CTE", + sql: "SELECT id FROM items", + rule: "SQLCLEAN034", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "logic in terminal select", + sql: "WITH final_rows AS (SELECT id FROM items) SELECT id FROM final_rows WHERE id > 0", + rule: "SQLCLEAN034", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "terminal select reads wrong CTE", + sql: "WITH first_rows AS (SELECT 1 AS id), final_rows AS (SELECT id FROM first_rows) SELECT id FROM first_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "plain terminal select", + sql: "WITH first_rows AS (SELECT 1 AS id), final_rows AS (SELECT id FROM first_rows) SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "plain terminal select with keyword-named columns", + sql: "WITH final_rows AS (SELECT 1 AS date, 2 AS comment, 3 AS key, 4 AS percent, 5 AS sequence, 6 AS time) SELECT date, comment, key, percent, sequence, time FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "terminal date literal remains an expression", + sql: "WITH final_rows AS (SELECT 1 AS id) SELECT DATE '2026-01-01' FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "terminal modulo calculation remains an expression", + sql: "WITH final_rows AS (SELECT 10 AS quantity, 3 AS pack_size) SELECT quantity % pack_size AS remainder FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "plain terminal select from CTE with output column list", + sql: "WITH final_rows(id) AS (SELECT 1) SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "CTE output column list names its calculation", + sql: "WITH final_rows(id) AS (SELECT 1) SELECT id FROM final_rows", + rule: "SQLCLEAN022", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "terminal union remains outside final CTE", + sql: "WITH final_rows AS (SELECT 1 AS id) SELECT id FROM final_rows UNION ALL SELECT id FROM final_rows", + rule: "SQLCLEAN034", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "terminal union is not a plain final CTE read", + sql: "WITH final_rows AS (SELECT 1 AS id) SELECT id FROM final_rows UNION ALL SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "parenthesized terminal union remains outside final CTE", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL (SELECT id FROM final_rows)", + rule: "SQLCLEAN034", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "parenthesized terminal union is not a plain final CTE read", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL (SELECT id FROM final_rows)", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "mixed parenthesized terminal union remains outside final CTE", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL SELECT id FROM final_rows", + rule: "SQLCLEAN034", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "mixed parenthesized terminal union is not a plain final CTE read", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) UNION ALL SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "mixed parenthesized terminal intersect is not a plain final CTE read", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) INTERSECT SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "mixed parenthesized terminal except is not a plain final CTE read", + sql: "WITH final_rows AS (SELECT 1 AS id) (SELECT id FROM final_rows) EXCEPT SELECT id FROM final_rows", + rule: "SQLCLEAN035", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "nested CTE", + sql: "SELECT id FROM (WITH nested_rows AS (SELECT 1 AS id) SELECT id FROM nested_rows)", + rule: "SQLCLEAN036", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "Snowflake hierarchy START WITH is not a nested CTE", + sql: "WITH entities AS (SELECT id, parent_id FROM source_rows), hierarchy AS (SELECT id FROM entities START WITH parent_id IS NULL CONNECT BY parent_id = PRIOR id) SELECT id FROM hierarchy", + rule: "SQLCLEAN036", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "recursive CTE", + sql: "WITH RECURSIVE numbers AS (SELECT 1 UNION ALL SELECT 2) SELECT * FROM numbers", + rule: "SQLCLEAN037", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "cross join", + sql: "SELECT left_rows.id FROM left_rows CROSS JOIN right_rows", + rule: "SQLCLEAN038", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "inline query after from", + sql: "SELECT derived.order_id FROM (SELECT order_id FROM orders) AS derived", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "inline query after join", + sql: "SELECT orders.order_id FROM orders INNER JOIN (SELECT order_id FROM shipments) AS shipped_orders ON orders.order_id = shipped_orders.order_id", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "inline values query after full outer join", + sql: "SELECT COALESCE(actual.order_id, expected.order_id) AS order_id FROM actual FULL OUTER JOIN (SELECT column1 AS order_id FROM VALUES (1), (2)) AS expected ON actual.order_id = expected.order_id", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "inline values relation after join", + sql: "SELECT orders.order_id FROM orders INNER JOIN (VALUES (1), (2)) AS expected(order_id) ON orders.order_id = expected.order_id", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "inline nested with query", + sql: "SELECT derived.order_id FROM (WITH imported_orders AS (SELECT order_id FROM orders) SELECT order_id FROM imported_orders) AS derived", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "redundantly parenthesized inline query", + sql: "SELECT derived.order_id FROM (((SELECT order_id FROM orders))) AS derived", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "comma-separated inline query", + sql: "SELECT orders.order_id FROM orders, (SELECT order_id FROM shipments) AS shipped_orders", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery", + sql: "SELECT (SELECT MAX(order_id) FROM orders) AS latest_order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery containing a derived relation", + sql: "SELECT (SELECT derived.order_id FROM (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) AS derived) AS order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "exists subquery containing a derived relation", + sql: "SELECT order_id FROM orders WHERE EXISTS (SELECT 1 FROM (SELECT nested.order_id FROM (SELECT order_id FROM shipments) AS nested) AS shipped_orders WHERE shipped_orders.order_id = orders.order_id)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "exists set branch containing a derived relation", + sql: "SELECT EXISTS (SELECT 1 UNION ALL SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar with query containing a derived relation in a CTE", + sql: "SELECT (WITH derived AS (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) SELECT order_id FROM derived) AS order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar with query containing a derived terminal relation", + sql: "SELECT (WITH orders AS (SELECT 1 AS order_id) SELECT derived.order_id FROM (SELECT order_id FROM orders) AS derived) AS order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "nested derived relations outside an excluded subquery", + sql: "SELECT derived.order_id FROM (SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested) AS derived", + rule: "SQLCLEAN039", + expected_count: 2, + }, + test_types::PlainSqlLintTestCase { + description: "nested derived relations inside a parenthesized ordinary relation", + sql: "SELECT derived.order_id FROM ((SELECT nested.order_id FROM (SELECT 1 AS order_id) AS nested)) AS derived", + rule: "SQLCLEAN039", + expected_count: 2, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery after projection comma", + sql: "SELECT 1 AS constant_value, (SELECT MAX(order_id) FROM orders) AS latest_order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery after expression from", + sql: "SELECT EXTRACT(YEAR FROM (SELECT MAX(order_date) FROM orders)) AS latest_order_year", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery after sibling projection subquery", + sql: "SELECT (SELECT MAX(order_id) FROM orders), COALESCE(NULL, (SELECT MAX(order_id) FROM orders))", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "expression from after sibling projection subquery", + sql: "SELECT (SELECT MAX(order_id) FROM orders), EXTRACT(YEAR FROM (SELECT MAX(order_date) FROM orders))", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery after order by comma", + sql: "SELECT order_id FROM orders ORDER BY order_id, (SELECT MAX(status_id) FROM statuses)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "scalar subquery with a derived relation after order by comma", + sql: "SELECT order_id FROM orders ORDER BY order_id, (SELECT derived.status_id FROM (SELECT 1 AS status_id) AS derived)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "exists subquery", + sql: "SELECT order_id FROM orders WHERE EXISTS (SELECT 1 FROM shipments WHERE shipments.order_id = orders.order_id)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "in subquery", + sql: "SELECT order_id FROM orders WHERE order_id IN (SELECT order_id FROM shipments)", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "named CTE relation", + sql: "WITH shipped_orders AS (SELECT order_id FROM shipments) SELECT orders.order_id FROM orders INNER JOIN shipped_orders ON orders.order_id = shipped_orders.order_id", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "materialized CTE containing a derived relation", + sql: "WITH orders AS MATERIALIZED (SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived) SELECT order_id FROM orders", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "not materialized CTE containing a derived relation", + sql: "WITH orders AS NOT MATERIALIZED (SELECT derived.order_id FROM (SELECT 1 AS order_id) AS derived) SELECT order_id FROM orders", + rule: "SQLCLEAN039", + expected_count: 1, + }, + test_types::PlainSqlLintTestCase { + description: "lateral query relation", + sql: "SELECT orders.order_id FROM orders, LATERAL (SELECT orders.order_id) AS derived", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "apply query relation", + sql: "SELECT orders.order_id FROM orders CROSS APPLY (SELECT orders.order_id) AS derived", + rule: "SQLCLEAN039", + expected_count: 0, + }, + test_types::PlainSqlLintTestCase { + description: "table function relation", + sql: "SELECT flattened.value FROM TABLE(FLATTEN(input => PARSE_JSON('[1]'))) AS flattened", + rule: "SQLCLEAN039", + expected_count: 0, + }, + ]; + + for test_case in test_cases { + let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + assert!( + diagnostics + .iter() + .all(|diagnostic| diagnostic["code"] == test_case.rule), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_native_rules_when_linting_then_diagnosis_and_remediation_are_actionable() +-> Result<(), String> { + let test_cases = [ + test_types::LintMetadataTestCase { + description: "NULL comparison guidance", + sql: "SELECT value FROM a WHERE value = NULL", + expected_code: "SQLCLEAN001", + expected_message: "Comparison with NULL is never true", + expected_remediation: "Use IS NULL or IS NOT NULL when testing for NULL.", + }, + test_types::LintMetadataTestCase { + description: "implicit join guidance", + sql: "SELECT a.id FROM a, b", + expected_code: "SQLCLEAN002", + expected_message: "Comma-separated sources create an implicit cartesian join", + expected_remediation: "Replace comma-separated sources with an explicit keyed join, or use CROSS JOIN when the cartesian product is intentional.", + }, + test_types::LintMetadataTestCase { + description: "unconditioned join guidance", + sql: "SELECT a.id FROM a JOIN b", + expected_code: "SQLCLEAN003", + expected_message: "Non-cross join has no meaningful condition", + expected_remediation: "Add a meaningful ON or USING condition, or declare an intentional cartesian product with CROSS JOIN.", + }, + test_types::LintMetadataTestCase { + description: "unordered limit guidance", + sql: "SELECT id FROM items LIMIT 1", + expected_code: "SQLCLEAN004", + expected_message: "Row selection is nondeterministic", + expected_remediation: "Add ORDER BY with a deterministic tie-breaker before LIMIT or OFFSET.", + }, + test_types::LintMetadataTestCase { + description: "unused CTE guidance", + sql: "WITH unused AS (SELECT 1) SELECT 1", + expected_code: "SQLCLEAN005", + expected_message: "CTE is unreachable from the final query", + expected_remediation: "Reference the CTE from the final query or another reachable CTE, or remove it.", + }, + test_types::LintMetadataTestCase { + description: "redundant distinct guidance", + sql: "SELECT DISTINCT id FROM items GROUP BY id", + expected_code: "SQLCLEAN006", + expected_message: "DISTINCT is redundant with the grouped output", + expected_remediation: "Remove DISTINCT; the equivalent GROUP BY already determines the output groups.", + }, + test_types::LintMetadataTestCase { + description: "positional set star guidance", + sql: "SELECT * FROM a UNION ALL SELECT * FROM b", + expected_code: "SQLCLEAN007", + expected_message: "Positional set operation is vulnerable to column-order drift", + expected_remediation: "Enumerate columns in the same order in every set-operation branch.", + }, + ]; + + for test_case in test_cases { + let diagnostics = helpers::diagnostics(test_case.sql)?; + let diagnostic = diagnostics + .first() + .ok_or_else(|| format!("{} should report a diagnostic", test_case.description))?; + assert_eq!( + diagnostic["code"], test_case.expected_code, + "{}", + test_case.description + ); + assert_eq!( + diagnostic["message"], test_case.expected_message, + "{}", + test_case.description + ); + assert_eq!( + diagnostic["remediation"], test_case.expected_remediation, + "{}", + test_case.description + ); + let start = diagnostic["start"].as_u64().unwrap_or_default(); + let end = diagnostic["end"].as_u64().unwrap_or_default(); + assert!( + end > start, + "{} should report a non-empty range", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_fixable_and_ambiguous_findings_when_linting_then_only_proven_edits_are_returned() +-> Result<(), String> { + let test_cases = [ + test_types::LintFixTestCase { + description: "right-hand NULL equality", + sql: "SELECT value FROM a WHERE value = NULL", + expected_code: "SQLCLEAN001", + expected_replacement: Some("IS"), + }, + test_types::LintFixTestCase { + description: "right-hand NULL inequality", + sql: "SELECT value FROM a WHERE value <> NULL", + expected_code: "SQLCLEAN001", + expected_replacement: Some("IS NOT"), + }, + test_types::LintFixTestCase { + description: "left-hand NULL comparison requires authored intent", + sql: "SELECT value FROM a WHERE NULL = value", + expected_code: "SQLCLEAN001", + expected_replacement: None, + }, + test_types::LintFixTestCase { + description: "plain conditionless join", + sql: "SELECT a.id FROM a JOIN b", + expected_code: "SQLCLEAN003", + expected_replacement: Some("CROSS JOIN"), + }, + test_types::LintFixTestCase { + description: "qualified conditionless join requires intent", + sql: "SELECT a.id FROM a LEFT JOIN b", + expected_code: "SQLCLEAN003", + expected_replacement: None, + }, + test_types::LintFixTestCase { + description: "conditionless semi join is never rewritten as cross join", + sql: "SELECT a.id FROM a SEMI JOIN b", + expected_code: "SQLCLEAN003", + expected_replacement: None, + }, + test_types::LintFixTestCase { + description: "redundant distinct", + sql: "SELECT DISTINCT id FROM items GROUP BY id", + expected_code: "SQLCLEAN006", + expected_replacement: Some(""), + }, + test_types::LintFixTestCase { + description: "single unused select CTE", + sql: "WITH unused AS (SELECT 1) SELECT 1", + expected_code: "SQLCLEAN005", + expected_replacement: Some(""), + }, + test_types::LintFixTestCase { + description: "reserved framework CTE is never deleted", + sql: "WITH __expected__items AS (SELECT 1) SELECT 1", + expected_code: "SQLCLEAN005", + expected_replacement: None, + }, + ]; + + for test_case in test_cases { + let diagnostics = helpers::diagnostics(test_case.sql)?; + let diagnostic = diagnostics + .iter() + .find(|item| item["code"] == test_case.expected_code) + .ok_or_else(|| format!("{} should report", test_case.description))?; + assert_eq!( + diagnostic["fix"]["replacement"].as_str(), + test_case.expected_replacement, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> Result<(), String> { + let test_cases = [ + test_types::AdditionalLintRuleTestCase { + description: "bare union", + sql: "SELECT 1 UNION SELECT 2", + rule: "SQLCLEAN008", + expected_anchor: Some("UNION"), + expected_replacement: Some("UNION DISTINCT"), + }, + test_types::AdditionalLintRuleTestCase { + description: "duplicate relation alias", + sql: "SELECT x.id FROM alpha AS x JOIN beta AS x ON x.id = x.id", + rule: "SQLCLEAN009", + expected_anchor: Some("x"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "duplicate output alias", + sql: "SELECT a AS value, b AS value FROM items", + rule: "SQLCLEAN010", + expected_anchor: Some("value"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "mixed grouping references", + sql: "SELECT a, b FROM items GROUP BY 1, b", + rule: "SQLCLEAN011", + expected_anchor: Some("GROUP"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "redundant else null", + sql: "SELECT CASE WHEN a THEN b ELSE NULL END FROM items", + rule: "SQLCLEAN012", + expected_anchor: Some("ELSE NULL"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "function-like distinct", + sql: "SELECT DISTINCT(a) FROM items", + rule: "SQLCLEAN013", + expected_anchor: Some("DISTINCT(a)"), + expected_replacement: Some("DISTINCT a"), + }, + test_types::AdditionalLintRuleTestCase { + description: "function-like distinct whose expression continues keeps its grouping", + sql: "SELECT DISTINCT(price + tax) * 2 AS total FROM orders", + rule: "SQLCLEAN013", + expected_anchor: Some("DISTINCT(price + tax)"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "constant scaffold predicate", + sql: "SELECT a FROM items WHERE 1 = 1", + rule: "SQLCLEAN014", + expected_anchor: Some("="), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "consecutive statement terminator", + sql: "SELECT 1;;", + rule: "SQLCLEAN015", + expected_anchor: Some(";"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "redundant self alias", + sql: "SELECT value AS value FROM items", + rule: "SQLCLEAN016", + expected_anchor: Some(" AS value"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified column self alias is redundant", + sql: "SELECT items.value AS value FROM items", + rule: "SQLCLEAN016", + expected_anchor: Some(" AS value"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified column self alias after an expression is redundant", + sql: "SELECT item_id + 1 AS next_item_id, items.value AS value FROM items", + rule: "SQLCLEAN016", + expected_anchor: Some(" AS value"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "top-modified column self alias is redundant", + sql: "SELECT TOP 5 value AS value FROM items", + rule: "SQLCLEAN016", + expected_anchor: Some(" AS value"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "terminal operand name does not make a calculation self-aliased", + sql: "SELECT price + value AS value FROM items", + rule: "SQLCLEAN016", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "cast type syntax is not an output self alias", + sql: "SELECT CAST(order_id::INTEGER AS INTEGER) AS order_id FROM orders", + rule: "SQLCLEAN016", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "relation self alias is not an output self alias", + sql: "SELECT orders.order_id FROM orders AS orders", + rule: "SQLCLEAN016", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "semi-structured path alias is not redundant", + sql: "SELECT event.payload:data:order_id AS order_id FROM events AS event", + rule: "SQLCLEAN016", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "constant row count", + sql: "SELECT COUNT(1) FROM items", + rule: "SQLCLEAN017", + expected_anchor: Some("1"), + expected_replacement: Some("*"), + }, + test_types::AdditionalLintRuleTestCase { + description: "unordered row number", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY id) FROM items", + rule: "SQLCLEAN018", + expected_anchor: Some("ROW_NUMBER"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "literal null in not-in list", + sql: "SELECT id FROM items WHERE id NOT IN (1, NULL)", + rule: "SQLCLEAN019", + expected_anchor: Some("NOT"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "set branch arity mismatch", + sql: "SELECT a, b FROM first UNION ALL SELECT c FROM second", + rule: "SQLCLEAN020", + expected_anchor: Some("UNION"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "uncontrolled projection star", + sql: "SELECT * FROM items", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "projection star without from remains uncontrolled", + sql: "SELECT *", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "mixed dynamic pivot projection remains uncontrolled", + sql: "SELECT *, customer_id FROM orders PIVOT(MAX(amount) FOR category IN (ANY))", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "static pivot star remains uncontrolled", + sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('books'))", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "static pivot any string remains uncontrolled", + sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('ANY'))", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "nested dynamic pivot does not exempt outer projection star", + sql: "SELECT * FROM orders WHERE EXISTS (SELECT 1 FROM items PIVOT(MAX(amount) FOR category IN (ANY)))", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unrelated any does not exempt static pivot star", + sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN ('books')) WHERE customer_id = ANY (SELECT customer_id FROM customers)", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "comma joined dynamic pivot star remains uncontrolled", + sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN (ANY)) AS p, customers AS c", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "joined audit relation parameter star remains uncontrolled", + sql: "SELECT * FROM __sqlclean_relation_parameter_0__ AS a CROSS JOIN customers AS c", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "terminal framework dependency star remains uncontrolled", + sql: "SELECT * FROM __sqlclean_interpolation_0__", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "transformed framework dependency import star remains uncontrolled", + sql: "WITH imported_orders AS (SELECT *, CURRENT_TIMESTAMP AS loaded_at FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "ordinary relation import star remains uncontrolled", + sql: "WITH imported_orders AS (SELECT * FROM orders) SELECT order_id FROM imported_orders", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "nested framework dependency import star remains uncontrolled", + sql: "WITH outer_query AS (WITH imported_orders AS (SELECT * FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders) SELECT order_id FROM outer_query", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased calculated projection", + sql: "SELECT price * quantity FROM items", + rule: "SQLCLEAN022", + expected_anchor: Some("price"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unused table alias", + sql: "SELECT id FROM items AS unused", + rule: "SQLCLEAN023", + expected_anchor: Some("AS unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "left join rejected in where", + sql: "SELECT a.id FROM a LEFT JOIN b AS right_side ON a.id = right_side.id WHERE right_side.active = TRUE", + rule: "SQLCLEAN024", + expected_anchor: Some("LEFT"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit inner join", + sql: "SELECT a.id FROM a JOIN b ON a.id = b.id", + rule: "SQLCLEAN025", + expected_anchor: Some("JOIN"), + expected_replacement: Some("INNER JOIN"), + }, + test_types::AdditionalLintRuleTestCase { + description: "mixed order directions", + sql: "SELECT a, b FROM items ORDER BY a, b DESC", + rule: "SQLCLEAN026", + expected_anchor: Some("ORDER"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unqualified multi-source column", + sql: "SELECT id FROM a JOIN b ON a.id = b.id", + rule: "SQLCLEAN027", + expected_anchor: Some("id"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "mixed single-source qualification", + sql: "SELECT items.id, name FROM items", + rule: "SQLCLEAN028", + expected_anchor: Some("name"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "plain macro slot reads count as unqualified beside a plain group read", + sql: "SELECT __sqlclean_macro_parameter_1__, __sqlclean_macro_parameter_0__.__sqlclean_macro_parameter_2__ AS orders FROM __sqlclean_macro_parameter_0__", + rule: "SQLCLEAN028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "a plain macro slot read beside a hand-qualified column is mixed", + sql: "SELECT __sqlclean_macro_parameter_0__.__sqlclean_macro_parameter_1__ AS orders, s.refunds FROM __sqlclean_macro_parameter_0__ AS s", + rule: "SQLCLEAN028", + expected_anchor: Some("__sqlclean_macro_parameter_0__"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unknown relation qualifier", + sql: "SELECT missing.id FROM items AS present", + rule: "SQLCLEAN029", + expected_anchor: Some("missing"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake shorthand cast type is not an unqualified column", + sql: "SELECT src.value::STRING AS value FROM items AS src", + rule: "SQLCLEAN028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake JSON path and shorthand cast type are not columns", + sql: "SELECT src.payload:item::TEXT AS item FROM items AS src", + rule: "SQLCLEAN028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake shorthand cast type is not a multi-source column", + sql: "SELECT first.value::TEXT AS value FROM first JOIN second ON first.id = second.id", + rule: "SQLCLEAN027", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit table alias is known in query scope", + sql: "SELECT present.id FROM items present", + rule: "SQLCLEAN029", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit joined alias contributes a selected value", + sql: "SELECT left_side.id, right_side.name FROM alpha left_side LEFT JOIN beta right_side ON left_side.id = right_side.id", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake lateral flatten comma is explicit lateral syntax", + sql: "SELECT flattened.value FROM items, LATERAL FLATTEN(input => items.payload) flattened", + rule: "SQLCLEAN002", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake VALUES row separator is not a relation separator", + sql: "SELECT column1 FROM VALUES (1), (2), (3)", + rule: "SQLCLEAN002", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "arithmetic multiplication is not a projected wildcard", + sql: "SELECT price * quantity AS total FROM items", + rule: "SQLCLEAN021", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "number-led multiplication is not a projected wildcard", + sql: "SELECT 100 * ratio AS percentage FROM items", + rule: "SQLCLEAN021", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "TOP projection wildcard remains controlled by star rule", + sql: "SELECT TOP 10 * FROM items", + rule: "SQLCLEAN021", + expected_anchor: Some("*"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified wildcard modifier does not require an alias", + sql: "SELECT src.* EXCLUDE (internal_id) FROM items AS src", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "EXISTS literal projection does not require an alias", + sql: "SELECT parent.id FROM parent WHERE EXISTS (SELECT 1 FROM child WHERE child.id = parent.id)", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "correlated subquery can reference an outer relation", + sql: "SELECT parent.id FROM parent WHERE EXISTS (SELECT 1 FROM child WHERE child.id = parent.id)", + rule: "SQLCLEAN029", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake JSON path key is not a relation qualifier", + sql: "SELECT PARSE_JSON(src.payload):item.name::VARCHAR AS item_name FROM items AS src", + rule: "SQLCLEAN029", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake JSON path starts from a qualified relation value", + sql: "SELECT parsed:item::VARCHAR AS item FROM items AS src, LATERAL FLATTEN(input => src.payload) AS parsed", + rule: "SQLCLEAN027", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "Snowflake ASOF JOIN alias is known in query scope", + sql: "SELECT matched.ts FROM requested request ASOF JOIN available matched MATCH_CONDITION (request.ts >= matched.ts) ON request.id = matched.id", + rule: "SQLCLEAN029", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "later set branches inherit the first branch output name", + sql: "SELECT 0 AS offset_value UNION ALL SELECT 1 UNION ALL SELECT 2", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "scalar subquery output does not require an alias inside its scope", + sql: "SELECT (SELECT MAX(child.value) FROM child WHERE child.id = parent.id) AS maximum_value FROM parent", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "later scalar function argument does not require an inner alias", + sql: "SELECT COALESCE((SELECT MAX(change_id) FROM first_changes), (SELECT MAX(change_id) FROM second_changes), 0) AS latest_change_id FROM support_tickets", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "scalar aggregate nested in conditional coalesce does not require an inner alias", + sql: "SELECT CASE WHEN batch.layout = 'fallback' THEN COALESCE((SELECT MAX(change.line_id) FROM first_changes AS change WHERE change.batch_id = batch.batch_id AND change.line_id < batch.line_id), (SELECT MAX(archive.line_id) FROM archived_changes AS archive WHERE archive.batch_id = batch.batch_id AND archive.line_id < batch.line_id), header.line_id) + 1 ELSE batch.start_line END AS start_line FROM support_batches AS batch LEFT JOIN support_headers AS header ON header.batch_id = batch.batch_id", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "prior nested relation does not contaminate a later scalar function argument", + sql: "WITH prior_changes AS (SELECT (SELECT MAX(change_id) FROM support_changes) AS latest_change_id FROM support_tickets), current_changes AS (SELECT COALESCE((SELECT MAX(change_id) FROM first_changes), (SELECT MAX(change_id) FROM second_changes), 0) AS latest_change_id FROM current_tickets) SELECT * FROM current_changes", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "scalar subquery after a prior CTE relation remains in projection scope", + sql: "WITH products_used AS (SELECT product_id FROM products), ticket_totals AS (SELECT ticket.ticket_id, (SELECT MAX(change_id) FROM support_changes) AS latest_change_id FROM support_tickets AS ticket) SELECT * FROM ticket_totals", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "comma-derived table calculation still requires an alias", + sql: "SELECT derived.value FROM items, (SELECT amount + tax FROM totals) AS derived", + rule: "SQLCLEAN022", + expected_anchor: Some("amount"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "scalar subquery scope does not cross a prior statement", + sql: "SELECT order_id FROM orders; SELECT invoice_id, (SELECT MAX(amount + tax) FROM totals) AS maximum_total FROM invoices", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "contextual keyword remains a simple qualified projection", + sql: "SELECT flattened.index FROM items AS src, LATERAL FLATTEN(input => src.payload) AS flattened", + rule: "SQLCLEAN022", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "generated expression sentinel is not an authored column", + sql: "SELECT first.id FROM first JOIN second ON first.id = second.id WHERE __sqlclean_interpolation_0__", + rule: "SQLCLEAN027", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "null-safe comparison FROM is not a relation clause", + sql: "SELECT CASE WHEN src.actual IS DISTINCT FROM src.expected THEN 1 ELSE 0 END AS differs FROM items AS src", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "earlier projection alias is not a relation column", + sql: "SELECT flattened.value AS item_index, item_index + 1 AS next_index FROM items AS src, LATERAL FLATTEN(input => src.payload) AS flattened", + rule: "SQLCLEAN027", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "current date context value is not a multi-source column", + sql: "SELECT first_side.id FROM first_side JOIN second_side ON first_side.id = second_side.id WHERE first_side.created_at >= CURRENT_DATE", + rule: "SQLCLEAN027", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "current date context value does not mix qualification styles", + sql: "SELECT first_side.id FROM first_side WHERE first_side.created_at >= CURRENT_DATE", + rule: "SQLCLEAN028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "simple boolean case", + sql: "SELECT CASE WHEN amount > 0 THEN TRUE ELSE FALSE END AS positive FROM items", + rule: "SQLCLEAN030", + expected_anchor: Some("CASE WHEN amount > 0 THEN TRUE ELSE FALSE END"), + expected_replacement: Some("COALESCE(amount > 0, FALSE)"), + }, + test_types::AdditionalLintRuleTestCase { + description: "unicode before simple boolean case preserves replacement text", + sql: "SELECT 'é', CASE WHEN amount > 0 THEN TRUE ELSE FALSE END AS positive FROM items", + rule: "SQLCLEAN030", + expected_anchor: Some("CASE WHEN amount > 0 THEN TRUE ELSE FALSE END"), + expected_replacement: Some("COALESCE(amount > 0, FALSE)"), + }, + test_types::AdditionalLintRuleTestCase { + description: "commented boolean case is diagnosed without a fix", + sql: "SELECT CASE WHEN amount /* reason */ > 0 THEN TRUE ELSE FALSE END FROM items", + rule: "SQLCLEAN030", + expected_anchor: Some("CASE WHEN amount /* reason */ > 0 THEN TRUE ELSE FALSE END"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "multi-branch boolean case is diagnosed without an unsafe partial fix", + sql: "SELECT CASE WHEN status = 'open' THEN TRUE WHEN priority = 'high' THEN TRUE ELSE FALSE END AS actionable FROM support_tickets", + rule: "SQLCLEAN030", + expected_anchor: Some( + "CASE WHEN status = 'open' THEN TRUE WHEN priority = 'high' THEN TRUE ELSE FALSE END", + ), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "boolean case whose branch continues after TRUE is not a plain conversion", + sql: "SELECT CASE WHEN is_paid THEN TRUE AND is_shipped ELSE FALSE END AS done FROM orders", + rule: "SQLCLEAN030", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "case nested directly in else", + sql: "SELECT CASE WHEN a THEN 1 ELSE CASE WHEN b THEN 2 END END FROM items", + rule: "SQLCLEAN031", + expected_anchor: Some("CASE"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "joined relation contributes no values", + sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id", + rule: "SQLCLEAN032", + expected_anchor: Some("JOIN"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "joined relation contributes through an aggregate expression", + sql: "SELECT ARRAY_AGG(b.value) AS values FROM a LEFT JOIN b ON a.id = b.id", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "bare projected star consumes every joined relation", + sql: "SELECT * FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unqualified projection prevents an unsupported unused relation claim", + sql: "SELECT customer_name FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unqualified aggregate argument prevents an unsupported unused relation claim", + sql: "SELECT MAX(customer_name) FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "left qualified star does not consume joined relation", + sql: "SELECT o.* FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", + rule: "SQLCLEAN032", + expected_anchor: Some("JOIN"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "cross joined relation contributes through a filter predicate", + sql: "SELECT orders.order_id FROM orders CROSS JOIN processing_cutoff AS cutoff WHERE orders.created_at < cutoff.created_at", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "lateral relation contributes through a downstream join condition", + sql: "SELECT orders.order_id, products.name FROM orders CROSS JOIN LATERAL UNNEST(orders.product_ids) AS item INNER JOIN products ON (products.product_id = item.product_id)", + rule: "SQLCLEAN032", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit union is clean", + sql: "SELECT 1 UNION ALL SELECT 2", + rule: "SQLCLEAN008", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "ordered row number without a declared key is unproven", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY id ORDER BY created_at, id) FROM items", + rule: "SQLCLEAN018", + expected_anchor: Some("ROW_NUMBER"), + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "equal set arity is clean", + sql: "SELECT a, b FROM first UNION ALL SELECT c, d FROM second", + rule: "SQLCLEAN020", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "dynamic pivot star has data-dependent columns", + sql: "SELECT * FROM orders PIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))", + rule: "SQLCLEAN021", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "audit relation parameter star has invocation-dependent columns", + sql: "SELECT * FROM __sqlclean_relation_parameter_0__", + rule: "SQLCLEAN021", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified alias is used", + sql: "SELECT kept.id FROM items AS kept", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "left join predicate retained in on is clean", + sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id AND b.active = TRUE", + rule: "SQLCLEAN024", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit inner join is clean", + sql: "SELECT a.id FROM a INNER JOIN b ON a.id = b.id", + rule: "SQLCLEAN025", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "null literal inside not-in subquery expression is not a literal list", + sql: "SELECT id FROM items WHERE id NOT IN (SELECT COALESCE(id, NULL) FROM other)", + rule: "SQLCLEAN019", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "left join null probe preserves unmatched rows", + sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id WHERE b.id IS NULL", + rule: "SQLCLEAN024", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "left join predicate guarded by null probe preserves unmatched rows", + sql: "SELECT a.id FROM a LEFT JOIN b ON a.id = b.id WHERE b.active = TRUE OR b.id IS NULL", + rule: "SQLCLEAN024", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified catalog path is not treated as an unknown relation alias", + sql: "SELECT catalog.schema.items.id FROM catalog.schema.items", + rule: "SQLCLEAN029", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "correlated subquery keeps outer table alias", + sql: "SELECT id FROM items AS kept WHERE EXISTS (SELECT 1 FROM other WHERE other.id = kept.id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "asof join is not rewritten as an inner join", + sql: "SELECT a.id FROM a ASOF JOIN b ON a.id = b.id", + rule: "SQLCLEAN025", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "semi join is not rewritten as an inner join", + sql: "SELECT a.id FROM a SEMI JOIN b ON a.id = b.id", + rule: "SQLCLEAN025", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "table alias carrying a column list is retained", + sql: "SELECT a, b FROM items AS t(a, b)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "bracket list commas do not change set projection arity", + sql: "SELECT a, [1, 2, 3] AS arr FROM t UNION ALL SELECT a, [4] AS arr FROM t", + rule: "SQLCLEAN020", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "quoted output alias is not redundant with an unquoted identifier", + sql: "SELECT value AS \"value\" FROM items", + rule: "SQLCLEAN016", + expected_anchor: None, + expected_replacement: None, + }, + ]; + + for test_case in test_cases { + let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; + assert_eq!( + diagnostics.len(), + usize::from(test_case.expected_anchor.is_some()), + "{}", + test_case.description + ); + let _ = test_case.expected_anchor.map(|expected_anchor| { + let diagnostic = &diagnostics[0]; + let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; + let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; + let source: String = test_case + .sql + .chars() + .skip(start) + .take(end - start) + .collect(); + assert_eq!(source, expected_anchor, "{}", test_case.description); + assert_eq!( + diagnostic["fix"]["replacement"].as_str(), + test_case.expected_replacement, + "{}", + test_case.description + ); + }); + } + Ok(()) +} + +#[test] +fn given_ceremonial_cte_contexts_when_linting_then_only_harness_dependencies_are_reachable() +-> Result<(), String> { + let test_cases = [ + test_types::CeremonialCteLintTestCase { + description: "macro harness outputs keep their helper dependency reachable", + sql: "WITH input_values AS (SELECT 7 AS amount), __macro_actual__ AS (SELECT amount FROM input_values), __macro_expected__ AS (SELECT 7 AS amount) SELECT 1", + expected_count: 0, + expected_start: None, + }, + test_types::CeremonialCteLintTestCase { + description: "non-harness double-underscore CTE remains diagnosable", + sql: "WITH __scratch AS (SELECT 1), __macro_actual__ AS (SELECT 7 AS amount), __macro_expected__ AS (SELECT 7 AS amount) SELECT 1", + expected_count: 1, + expected_start: Some(5), + }, + ]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQLCLEAN005"], + "fixture_harness": fixture_harness(), + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or_else(|| "diagnostics should be an array".to_string())?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + diagnostics + .first() + .and_then(|diagnostic| diagnostic["start"].as_u64()), + test_case.expected_start, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_dependency_identity_when_linting_import_stars_then_only_plain_top_level_import_is_accepted() +-> Result<(), String> { + let test_cases = [ + test_types::DependencyImportLintTestCase { + description: "plain top-level dependency import", + sql: "WITH imported_orders AS (SELECT * FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 0, + }, + test_types::DependencyImportLintTestCase { + description: "star without from remains diagnosable", + sql: "WITH imported_orders AS (SELECT *) SELECT order_id FROM imported_orders", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 1, + }, + test_types::DependencyImportLintTestCase { + description: "set operation import is transformed", + sql: "WITH imported_orders AS (SELECT * FROM __sqlclean_interpolation_0__ UNION ALL SELECT order_id FROM orders) SELECT order_id FROM imported_orders", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 1, + }, + test_types::DependencyImportLintTestCase { + description: "dynamic relation is not a dependency", + sql: "WITH imported_orders AS (SELECT * FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders", + dependency_identifiers: &[], + expected_count: 1, + }, + test_types::DependencyImportLintTestCase { + description: "terminal dependency star remains uncontrolled", + sql: "SELECT * FROM __sqlclean_interpolation_0__", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 1, + }, + test_types::DependencyImportLintTestCase { + description: "transformed dependency import remains uncontrolled", + sql: "WITH imported_orders AS (SELECT *, CURRENT_TIMESTAMP AS loaded_at FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 1, + }, + test_types::DependencyImportLintTestCase { + description: "nested dependency import remains uncontrolled", + sql: "WITH outer_query AS (WITH imported_orders AS (SELECT * FROM __sqlclean_interpolation_0__) SELECT order_id FROM imported_orders) SELECT order_id FROM outer_query", + dependency_identifiers: &["__sqlclean_interpolation_0__"], + expected_count: 1, + }, + ]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQLCLEAN021"], + "dependency_identifiers": test_case.dependency_identifiers, + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or_else(|| "diagnostics should be an array".to_owned())?; + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_ceremonial_select_context_when_linting_terminal_shape_then_rule_035_allows_it() +-> Result<(), String> { + let test_cases = [test_types::PlainSqlLintTestCase { + description: "direct ceremonial select is valid in harness context", + sql: "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id) SELECT 1", + rule: "SQLCLEAN035", + expected_count: 0, + }]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": [test_case.rule], + "fixture_harness": fixture_harness(), + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or_else(|| "diagnostics should be an array".to_owned())?; + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +#[ignore = "lint is unavailable for tsql until SQLQuack parses it"] +fn given_tsql_bare_union_when_linting_then_diagnostic_has_no_invalid_fix() -> Result<(), String> { + let test_cases = [test_types::DialectLintRuleTestCase { + description: "T-SQL bare UNION is diagnosed without an invalid fix", + sql: "SELECT 1 UNION SELECT 2", + dialect: "tsql", + rule: "SQLCLEAN008", + expected_count: 1, + expected_fix: false, + expected_reason: "the active dialect does not accept explicit UNION DISTINCT", + }]; + + for test_case in test_cases { + let diagnostics = + helpers::diagnostics_for_dialect(test_case.sql, test_case.dialect, &[test_case.rule])?; + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + diagnostics[0].get("fix").is_some(), + test_case.expected_fix, + "{}", + test_case.description + ); + assert_eq!( + diagnostics[0]["fix_unavailable_reason"], test_case.expected_reason, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_unknown_rule_when_linting_then_request_is_rejected() -> Result<(), String> { + let test_cases = [test_types::InvalidRuleTestCase { + description: "unknown native lint rule", + rule: "SQLCLEAN999", + expected_message: "unknown native lint rule 'SQLCLEAN999'", + }]; + + for test_case in &test_cases { + let request = json!({"version": 1, "sql": "SELECT 1", "dialect": "snowflake", "enabled_rules": [test_case.rule]}).to_string(); + let error = lint_json(&request) + .err() + .ok_or_else(|| "unknown rule should fail".to_string())?; + assert!( + error.contains(test_case.expected_message), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_dialect_sqlquack_does_not_parse_when_linting_then_lint_is_unavailable_not_clean() +-> Result<(), String> { + let test_cases = [ + test_types::UnavailableDialectTestCase { + description: "a known dialect without a SQLQuack parser is unavailable, not clean", + dialect: "tsql", + expected_unavailable: Some( + "SQL lint is unavailable for the tsql dialect: SQLQuack does not parse it yet", + ), + expected_diagnostics: 0, + }, + test_types::UnavailableDialectTestCase { + description: "a parsed dialect is linted", + dialect: "snowflake", + expected_unavailable: None, + expected_diagnostics: 1, + }, + ]; + for test_case in &test_cases { + let request = json!({"version": 1, "sql": "SELECT a FROM t WHERE a = NULL", "dialect": test_case.dialect}); + let payload: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!( + payload["unavailable"].as_str(), + test_case.expected_unavailable, + "{}", + test_case.description + ); + assert_eq!( + payload["diagnostics"].as_array().map_or(0, Vec::len), + test_case.expected_diagnostics, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_unknown_dialect_when_linting_then_request_is_rejected() { + let test_cases = [test_types::UnknownDialectTestCase { + description: "a name no engine knows is an error, not unavailable lint", + dialect: "warehouse", + expected_message: "Unknown dialect: warehouse", + }]; + for test_case in &test_cases { + let request = json!({"version": 1, "sql": "SELECT 1", "dialect": test_case.dialect}); + let error = lint_json(&request.to_string()).err().unwrap_or_default(); + assert!( + error.contains(test_case.expected_message), + "{}: {error}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_types.rs b/crates/sqlquack-lint/src/lint/tests/test_types.rs new file mode 100644 index 000000000..96e362b80 --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_types.rs @@ -0,0 +1,154 @@ +pub(crate) struct LintTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_codes: &'static [&'static str], + pub expected_anchors: &'static [(&'static str, &'static str)], +} + +pub(crate) struct EmptyFixtureLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) dialect: &'static str, + pub(crate) in_fixture_harness: bool, + pub(crate) expected_diagnostic_count: usize, +} + +pub(crate) struct BatchLintTestCase { + pub(crate) description: &'static str, + pub(crate) queries: &'static [&'static str], + pub(crate) expected_responses: usize, +} + +pub(crate) struct LintMetadataTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_code: &'static str, + pub expected_message: &'static str, + pub expected_remediation: &'static str, +} + +pub(crate) struct LintFixTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_code: &'static str, + pub expected_replacement: Option<&'static str>, +} + +pub(crate) struct AdditionalLintRuleTestCase { + pub description: &'static str, + pub sql: &'static str, + pub rule: &'static str, + pub expected_anchor: Option<&'static str>, + pub expected_replacement: Option<&'static str>, +} + +pub(crate) struct CeremonialCteLintTestCase { + pub description: &'static str, + pub sql: &'static str, + pub expected_count: usize, + pub expected_start: Option, +} + +pub(crate) struct PlainSqlLintTestCase { + pub description: &'static str, + pub sql: &'static str, + pub rule: &'static str, + pub expected_count: usize, +} + +pub(crate) struct SqlMacroOutputLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + /// The text a `@sql.` call generated, and the names the call passed. + pub(crate) output: &'static str, + pub(crate) relations: &'static [&'static str], + pub(crate) expected_count: usize, +} + +pub(crate) struct DynamicOutputStarLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_count: usize, + pub(crate) expected_prefix: &'static str, +} + +pub(crate) struct DependencyImportLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) dependency_identifiers: &'static [&'static str], + pub(crate) expected_count: usize, +} + +pub(crate) struct DialectLintRuleTestCase { + pub description: &'static str, + pub sql: &'static str, + pub dialect: &'static str, + pub rule: &'static str, + pub expected_count: usize, + pub expected_fix: bool, + pub expected_reason: &'static str, +} + +pub(crate) struct InvalidRuleTestCase { + pub description: &'static str, + pub rule: &'static str, + pub expected_message: &'static str, +} + +pub(crate) struct FunctionDepthTestCase { + pub description: &'static str, + pub depth: usize, + pub expected_diagnostic_count: usize, +} + +pub(crate) struct FunctionDepthFailureTestCase { + pub description: &'static str, + pub depth: usize, + pub expected_code: &'static str, + pub expected_limit: &'static str, +} + +pub(crate) struct NestingRefusalTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub depth: usize, + pub expected_code: &'static str, +} + +pub(crate) struct CrossJoinTestCase { + pub description: &'static str, + pub sql: &'static str, + pub dialect: &'static str, + pub one_row_relations: &'static [&'static str], + pub expected_messages: &'static [&'static str], +} + +pub(crate) struct CrossJoinSubjectTestCase { + pub description: &'static str, + pub message: &'static str, + pub expected_subject: Option<(&'static str, Option<&'static str>)>, +} + +pub(crate) struct UnavailableDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) expected_unavailable: Option<&'static str>, + pub(crate) expected_diagnostics: usize, +} + +pub(crate) struct UnknownDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) expected_message: &'static str, +} + +pub(crate) struct LongChainTestCase { + pub description: &'static str, + pub dialect: &'static str, + /// The statement, with `{chain}` standing for `terms` copies of `term` joined by `separator`. + pub template: &'static str, + pub term: &'static str, + pub separator: &'static str, + pub terms: usize, + pub expected_lints: bool, +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_unused_table_alias.rs b/crates/sqlquack-lint/src/lint/tests/test_unused_table_alias.rs new file mode 100644 index 000000000..0904732cd --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_unused_table_alias.rs @@ -0,0 +1,317 @@ +use crate::lint::tests::{helpers, test_types}; + +#[test] +fn given_implicit_and_explicit_table_aliases_when_linting_then_unused_aliases_match() +-> Result<(), String> { + let test_cases = [ + test_types::AdditionalLintRuleTestCase { + description: "unused implicit table alias", + sql: "SELECT id FROM items unused", + rule: "SQLCLEAN023", + expected_anchor: Some("unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "unused implicit joined table alias", + sql: "SELECT o.id FROM orders o INNER JOIN customers unused ON o.customer_id = o.id", + rule: "SQLCLEAN023", + expected_anchor: Some("unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "unused implicit qualified table alias", + sql: "SELECT id FROM warehouse.analytics.items unused", + rule: "SQLCLEAN023", + expected_anchor: Some("unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "unused implicit subquery alias", + sql: "SELECT id FROM (SELECT id FROM items) unused", + rule: "SQLCLEAN023", + expected_anchor: Some("unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "unused implicit lateral table function alias", + sql: "SELECT src.id FROM items src, LATERAL FLATTEN(input => src.payload) unused", + rule: "SQLCLEAN023", + expected_anchor: Some("unused"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "used implicit table alias", + sql: "SELECT kept.id FROM items kept", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "used implicit joined table aliases", + sql: "SELECT o.id, c.name FROM orders o INNER JOIN customers c ON o.customer_id = c.id", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "used implicit qualified table alias", + sql: "SELECT kept.id FROM warehouse.analytics.items kept", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "used implicit subquery alias", + sql: "SELECT sub.id FROM (SELECT id FROM items) sub", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "used implicit lateral table function alias", + sql: "SELECT flattened.value FROM items src, LATERAL FLATTEN(input => src.payload) flattened", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias used by correlated subquery", + sql: "SELECT id FROM items kept WHERE EXISTS (SELECT 1 FROM other WHERE other.id = kept.id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias carrying a column list is retained", + sql: "SELECT a FROM items t(a)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "qualified relation name is not an implicit alias", + sql: "SELECT id FROM warehouse.analytics.items", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "join modifier keywords are not implicit aliases", + sql: "SELECT o.id, c.name FROM orders o POSITIONAL JOIN customers c", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "join condition identifiers are not implicit aliases", + sql: "SELECT o.id, c.name FROM orders o INNER JOIN customers c USING (customer_id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + ]; + + for test_case in test_cases { + let diagnostics = helpers::diagnostics_for_rules(test_case.sql, &[test_case.rule])?; + assert_eq!( + diagnostics.len(), + usize::from(test_case.expected_anchor.is_some()), + "{}", + test_case.description + ); + let _ = test_case.expected_anchor.map(|expected_anchor| { + let diagnostic = &diagnostics[0]; + let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; + let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; + assert_eq!( + &test_case.sql[start..end], + expected_anchor, + "{}", + test_case.description + ); + assert_eq!( + diagnostic["fix"]["replacement"].as_str(), + test_case.expected_replacement, + "{}", + test_case.description + ); + }); + } + Ok(()) +} + +#[test] +fn given_join_modifiers_and_whole_row_uses_when_linting_duckdb_then_aliases_are_kept() +-> Result<(), String> { + let test_cases = [ + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before positional join has no alias", + sql: "SELECT * FROM orders POSITIONAL JOIN customers", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before asof join has no alias", + sql: "SELECT * FROM orders ASOF JOIN prices ON orders.item_id = prices.item_id AND orders.ordered_at >= prices.valid_from", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before natural join has no alias", + sql: "SELECT * FROM orders NATURAL JOIN customers", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before anti join has no alias", + sql: "SELECT * FROM orders ANTI JOIN refunds USING (order_id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before semi join has no alias", + sql: "SELECT * FROM orders SEMI JOIN refunds USING (order_id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before cross join has no alias", + sql: "SELECT * FROM orders CROSS JOIN customers", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias repeating the relation name is unused", + sql: "SELECT id FROM orders orders", + rule: "SQLCLEAN023", + expected_anchor: Some("orders"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit alias repeating the relation name is unused", + sql: "SELECT id FROM orders AS orders", + rule: "SQLCLEAN023", + expected_anchor: Some("AS orders"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "alias repeating the relation name is used when referenced", + sql: "SELECT orders.id FROM orders orders", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before window clause has no alias", + sql: "SELECT SUM(amount) OVER w FROM orders WINDOW w AS (ORDER BY id)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before tablesample has no alias", + sql: "SELECT * FROM orders TABLESAMPLE BERNOULLI (10)", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unaliased relation before unpivot has no alias", + sql: "SELECT * FROM orders UNPIVOT (amount FOR month IN (jan, feb))", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias used as whole-row function argument", + sql: "SELECT row_to_json(o) AS payload FROM orders o", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit alias used as whole-row function argument", + sql: "SELECT row_to_json(o) AS payload FROM orders AS o", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias used as bare projection", + sql: "SELECT o FROM orders o", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit alias used as bare projection", + sql: "SELECT o FROM orders AS o", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "implicit alias used in whole-row comparison", + sql: "SELECT 1 AS present FROM orders o WHERE o IS NOT NULL", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "explicit alias used in whole-row comparison", + sql: "SELECT 1 AS present FROM orders AS o WHERE o IS NOT NULL", + rule: "SQLCLEAN023", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "unused implicit alias beside an unrelated qualified column is reported", + sql: "SELECT c.o FROM customers c CROSS JOIN orders o", + rule: "SQLCLEAN023", + expected_anchor: Some("o"), + expected_replacement: Some(""), + }, + test_types::AdditionalLintRuleTestCase { + description: "unused explicit alias shadowed by an output alias is reported", + sql: "SELECT order_id AS o FROM orders AS o", + rule: "SQLCLEAN023", + expected_anchor: Some("AS o"), + expected_replacement: Some(""), + }, + ]; + + for test_case in test_cases { + let diagnostics = + helpers::diagnostics_for_dialect(test_case.sql, "duckdb", &[test_case.rule])?; + assert_eq!( + diagnostics.len(), + usize::from(test_case.expected_anchor.is_some()), + "{}", + test_case.description + ); + let _ = test_case.expected_anchor.map(|expected_anchor| { + let diagnostic = &diagnostics[0]; + let start = diagnostic["start"].as_u64().unwrap_or_default() as usize; + let end = diagnostic["end"].as_u64().unwrap_or_default() as usize; + assert_eq!( + &test_case.sql[start..end], + expected_anchor, + "{}", + test_case.description + ); + assert_eq!( + diagnostic["fix"]["replacement"].as_str(), + test_case.expected_replacement, + "{}", + test_case.description + ); + }); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/cte_columns.rs b/crates/sqlquack-lint/src/quality/_helpers/cte_columns.rs new file mode 100644 index 000000000..9994395d2 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/cte_columns.rs @@ -0,0 +1,480 @@ +//! SQLCLEAN042: output columns of non-final CTEs that no later step reads. + +use std::borrow::Cow; + +use crate::tree::main::walk_in_scope::walk_in_scope; +use sqlquack_syntax::expressions::queries::models::Cte; +use sqlquack_syntax::expressions::queries::models::JoinKind; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::expressions::queries::models::With; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::lint::models::{LintDiagnostic, LintEdit}; +use crate::quality::constants::{ + DISTINCT_UNUSED_REMEDIATION, GROUP_BY_ALL_UNUSED_REMEDIATION, GROUP_KEY_UNUSED_REMEDIATION, + UNUSED_CTE_OUTPUT, +}; +use crate::quality::models::QualityRequest; +use crate::quality::projection_spans::projection_spans; +use crate::quality::removal_safety::{expression_refusal, reads_whole_rows, select_refusal}; +use crate::quality::syntax::{ + Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier, + sources, +}; +use rustc_hash::{FxHashMap, FxHashSet}; + +const HARNESS_PREFIX: &str = "__"; +const COLUMNS_FUNCTION: &str = "COLUMNS"; +const LAYOUT_FIX_REFUSAL: &str = "The projection list could not be mapped to authored tokens"; + +/// What one later step reads from a CTE. +#[derive(Default)] +struct Usage { + names: FxHashSet, + everything: bool, + passes_to: Vec, + read: bool, +} + +impl Usage { + fn merged(mut self, other: Self) -> Self { + self.names.extend(other.names); + self.everything |= other.everything; + self.passes_to.extend(other.passes_to); + self.read |= other.read; + self + } +} + +/// One CTE whose unused outputs are being reported. +struct CteOutputs<'a> { + cte: &'a Cte, + name: &'a str, + usage: Option<&'a Usage>, +} + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + if request.fixture_body { + return Vec::new(); + } + let mut found: Vec = Vec::new(); + for statement in request.statements { + let Expression::Select(root) = statement else { + continue; + }; + let Some(with) = root.with.as_ref().filter(|with| !with.recursive) else { + continue; + }; + let names: Vec = with.ctes.iter().map(|cte| lower(&cte.alias.name)).collect(); + let used: FxHashMap = + reads(&with.ctes, &names, terminal_nodes(statement, with)); + let select_tokens: FxHashMap = cte_select_tokens(request.tokens); + for (index, cte) in with + .ctes + .iter() + .enumerate() + .take(with.ctes.len().saturating_sub(1)) + { + let name: &str = &names[index]; + if name.starts_with(HARNESS_PREFIX) || request.externally_referenced_ctes.contains(name) + { + continue; + } + found.extend(unused_outputs( + request, + &select_tokens, + CteOutputs { + cte, + name, + usage: used.get(name), + }, + )); + } + } + found +} + +/// Names each CTE's later readers use, following `SELECT *` pass-through to the reader. +fn reads<'a>( + ctes: &'a [Cte], + names: &[String], + terminal: Vec<&'a Expression>, +) -> FxHashMap { + let bodies: Vec<(Option, Vec<&Expression>)> = ctes + .iter() + .map(|cte| (Some(lower(&cte.alias.name)), cte.this.dfs().collect())) + .chain(std::iter::once((None, terminal))) + .collect(); + let known: FxHashSet<&str> = names.iter().map(String::as_str).collect(); + let outputs: Vec>> = ctes.iter().map(output_names).collect(); + let mut readers: FxHashMap> = FxHashMap::default(); + for (index, (_, body)) in bodies.iter().enumerate() { + for name in read_relations(body) { + if known.contains(name.as_str()) { + readers.entry(name).or_default().push(index); + } + } + } + let mut usage: FxHashMap = FxHashMap::default(); + for (index, name) in names.iter().enumerate() { + let mut current: Usage = Usage::default(); + if index + 1 == names.len() { + current.everything = true; + } + for &reader in readers.get(name.as_str()).into_iter().flatten() { + if reader > index { + let (reader_name, body) = &bodies[reader]; + let target: Target<'_> = Target { + name, + outputs: outputs[index].as_ref(), + }; + current = current.merged(reader_usage(&target, reader_name.clone(), body)); + } + } + current.names.extend(self_reads(&ctes[index].this)); + usage.insert(name.clone(), current); + } + for index in (0..names.len()).rev() { + let passes: Vec = usage[&names[index]].passes_to.clone(); + for reader in passes { + let (everything, inherited): (bool, Vec) = + usage.get(&reader).map_or((true, Vec::new()), |next| { + (next.everything, next.names.iter().cloned().collect()) + }); + if let Some(current) = usage.get_mut(&names[index]) { + current.everything |= everything; + current.names.extend(inherited); + } + } + } + usage +} + +/// The output names of a CTE, or None when a star or unnamed projection hides them. +fn output_names(cte: &Cte) -> Option> { + let Expression::Select(select) = &cte.this else { + return None; + }; + projections(select) + .into_iter() + .map(|item| match item { + Projection::Named { name, .. } => Some(name), + _ => None, + }) + .collect() +} + +/// The statement's query without its WITH clause, depth first and borrowed, not recursively cloned. +fn terminal_nodes<'a>(statement: &'a Expression, with: &'a With) -> Vec<&'a Expression> { + let skipped: Vec<&Expression> = with + .ctes + .iter() + .map(|cte| &cte.this) + .chain(with.search.as_deref()) + .collect(); + let mut nodes: Vec<&Expression> = vec![statement]; + for child in statement.children() { + if !skipped.iter().any(|body| std::ptr::eq(*body, child)) { + nodes.extend(child.dfs()); + } + } + nodes +} + +fn read_relations(body: &[&Expression]) -> FxHashSet { + body.iter() + .filter_map(|node| match node { + Expression::Table(table) => Some(lower(&table.name.name)), + _ => None, + }) + .collect() +} + +/// The CTE whose readers are being inspected. +struct Target<'a> { + name: &'a str, + outputs: Option<&'a FxHashSet>, +} + +/// How one reader, given as its nodes depth first from its root, uses the target CTE. +fn reader_usage(target: &Target<'_>, reader: Option, body: &[&Expression]) -> Usage { + let name: &str = target.name; + let mut usage: Usage = Usage::default(); + let mut aliases: FxHashSet = FxHashSet::default(); + let mut top_level_star: bool = false; + for &node in body { + let Expression::Select(select) = node else { + continue; + }; + let local: Vec = sources(select) + .into_iter() + .filter(|(_, relation, _)| relation.as_deref() == Some(name)) + .map(|(alias, _, _)| alias) + .collect(); + if local.is_empty() { + continue; + } + if select.joins.iter().any(|join| { + matches!( + join.kind, + JoinKind::Natural + | JoinKind::NaturalLeft + | JoinKind::NaturalRight + | JoinKind::NaturalFull + ) + }) { + usage.everything = true; + } + for join in &select.joins { + usage + .names + .extend(join.using.iter().map(|identifier| lower(&identifier.name))); + } + let top: bool = body.first().is_some_and(|root| std::ptr::eq(node, *root)); + if reads_as_row(select, &local, target.outputs) { + usage.everything = true; + } + for projection in projections(select) { + if let Projection::Star(star) = projection + && star + .table + .as_ref() + .is_none_or(|table| local.contains(&lower(&table.name))) + { + if top + && reader.is_some() + && star.except.is_none() + && star.replace.is_none() + && !reads_whole_rows(select) + { + top_level_star = true; + } else { + usage.everything = true; + } + } + } + aliases.extend(local); + } + if aliases.is_empty() { + return usage; + } + usage.read = true; + usage.everything |= body.iter().any( + |node| matches!(node, Expression::Column(column) if is_interpolation(&column.name.name)), + ); + if top_level_star && let Some(reader) = reader { + usage.passes_to.push(reader); + } + for column in body.iter().filter_map(|node| match node { + Expression::Column(column) => Some(column), + _ => None, + }) { + if qualifier(column).is_none_or(|table| aliases.contains(&table)) { + usage.names.insert(lower(&column.name.name)); + } + } + usage +} + +/// Whether a reader consumes whole rows: a nested star, `COLUMNS(...)`, or the alias itself. +fn reads_as_row(select: &Select, local: &[String], outputs: Option<&FxHashSet>) -> bool { + for clause in clause_roots(select) { + for node in walk_in_scope(clause) { + if reads_row_at(select, node, local, outputs) { + return true; + } + } + } + false +} + +fn reads_row_at( + select: &Select, + node: &Expression, + local: &[String], + outputs: Option<&FxHashSet>, +) -> bool { + match node { + Expression::Star(star) => { + let projected: bool = select + .expressions + .iter() + .any(|item| std::ptr::eq(item, node)); + let table: Option = star.table.as_ref().map(|table| lower(&table.name)); + !projected && table.is_none_or(|table| local.contains(&table)) + } + Expression::BracedWildcard(_) => true, + Expression::Function(function) => function.name.eq_ignore_ascii_case(COLUMNS_FUNCTION), + Expression::Column(column) => { + let read: String = lower(&column.name.name); + let named_output: bool = outputs.is_some_and(|names| names.contains(&read)); + column.table.is_none() && local.contains(&read) && !named_output + } + _ => false, + } +} + +/// Root expressions of every clause of one select, without its FROM relations. +fn clause_roots(select: &Select) -> Vec<&Expression> { + let mut roots: Vec<&Expression> = select.expressions.iter().collect(); + roots.extend(select.where_clause.iter().map(|clause| &clause.this)); + roots.extend(select.having.iter().map(|clause| &clause.this)); + roots.extend(select.qualify.iter().map(|clause| &clause.this)); + if let Some(group) = &select.group_by { + roots.extend(group.expressions.iter()); + } + if let Some(order) = &select.order_by { + roots.extend(order.expressions.iter().map(|ordered| &ordered.this)); + } + roots.extend(select.joins.iter().filter_map(|join| join.on.as_ref())); + roots +} + +/// Lateral alias reuse: a later projection or a clause of the CTE reading an earlier output. +fn self_reads(body: &Expression) -> FxHashSet { + let mut reads: FxHashSet = FxHashSet::default(); + let Expression::Select(select) = body else { + return reads; + }; + let mut earlier: FxHashSet = FxHashSet::default(); + for item in projections(select) { + if let Projection::Named { name, expression } = item { + for read in unqualified_reads(expression) { + if earlier.contains(&read) && read != name { + reads.insert(read); + } + } + earlier.insert(name); + } + } + let mut clauses: Vec<&Expression> = Vec::new(); + clauses.extend(select.where_clause.iter().map(|clause| &clause.this)); + clauses.extend( + select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()), + ); + clauses.extend(select.having.iter().map(|clause| &clause.this)); + clauses.extend(select.qualify.iter().map(|clause| &clause.this)); + if let Some(order) = &select.order_by { + clauses.extend(order.expressions.iter().map(|ordered| &ordered.this)); + } + for clause in clauses { + reads.extend( + unqualified_reads(clause) + .into_iter() + .filter(|read| earlier.contains(read)), + ); + } + reads +} + +fn unqualified_reads(expression: &Expression) -> Vec { + columns(expression) + .into_iter() + .filter(|column| qualifier(column).is_none()) + .map(|column| lower(&column.name.name)) + .collect() +} + +fn unused_outputs( + request: &QualityRequest<'_>, + select_tokens: &FxHashMap, + outputs: CteOutputs<'_>, +) -> Vec { + let CteOutputs { cte, name, usage } = outputs; + let Expression::Select(select) = &cte.this else { + return Vec::new(); + }; + let Some(usage) = usage.filter(|usage| !usage.everything && usage.read) else { + return Vec::new(); + }; + if !cte.columns.is_empty() { + return Vec::new(); + } + let output: Vec> = projections(select); + if !output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + return Vec::new(); + } + let spans: Vec<(usize, usize)> = select_tokens + .get(name) + .map(|&select_token| projection_spans(request.tokens, select_token)) + .unwrap_or_default(); + let grouped: FxHashSet = select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()) + .flat_map(columns) + .map(|column| lower(&column.name.name)) + .collect(); + let select_refused: Option<&'static str> = select_refusal(select, output.len()); + let mut found: Vec = Vec::new(); + for (index, item) in output.iter().enumerate() { + let Projection::Named { + name: column, + expression, + } = item + else { + continue; + }; + if usage.names.contains(column) || is_interpolation(column) { + continue; + } + let grouping_key: bool = matches!(expression, Expression::Column(source) if grouped.contains(&lower(&source.name.name))); + let (start, end) = spans + .get(index) + .copied() + .filter(|_| spans.len() == output.len()) + .unwrap_or((0, 0)); + let refusal: Option<&'static str> = select_refused + .or_else(|| expression_refusal(expression)) + .or_else(|| (spans.len() != output.len()).then_some(LAYOUT_FIX_REFUSAL)); + let fix: Option = refusal + .is_none() + .then(|| projection_removal(&spans, index)) + .flatten() + .map(|(start, end)| LintEdit { + start, + end, + replacement: String::new(), + }); + found.push(LintDiagnostic { + code: UNUSED_CTE_OUTPUT.code, + message: Cow::Borrowed(UNUSED_CTE_OUTPUT.message), + remediation: Cow::Borrowed(if select.distinct { + DISTINCT_UNUSED_REMEDIATION + } else if select + .group_by + .as_ref() + .is_some_and(|group| group.all == Some(true)) + { + GROUP_BY_ALL_UNUSED_REMEDIATION + } else if grouping_key { + GROUP_KEY_UNUSED_REMEDIATION + } else { + UNUSED_CTE_OUTPUT.remediation + }), + start, + end, + fix_unavailable_reason: if fix.is_none() { + Some(refusal.unwrap_or(LAYOUT_FIX_REFUSAL)) + } else { + None + }, + fix, + }); + } + found +} + +/// Placeholders for template expressions can stand for any columns. +fn is_interpolation(name: &str) -> bool { + let name: String = lower(name); + crate::constants::GENERATED_PLACEHOLDER_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/json_parses.rs b/crates/sqlquack-lint/src/quality/_helpers/json_parses.rs new file mode 100644 index 000000000..7addad039 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/json_parses.rs @@ -0,0 +1,209 @@ +//! SQLCLEAN045: the same JSON payload column parsed more than once in one SELECT scope. + +use std::borrow::Cow; +use std::collections::HashMap; + +use crate::tree::main::call_name::call_name; +use crate::tree::main::walk_in_scope::walk_in_scope; +use sqlquack_syntax::expressions::names::models::Column; +use sqlquack_syntax::expressions::names::models::Identifier; +use sqlquack_syntax::tokens::models::Token; +use sqlquack_syntax::{Dialect, Expression, ExpressionWalk}; + +use crate::lint::models::LintDiagnostic; +use crate::quality::constants::REPEATED_JSON_PARSE; +use crate::quality::models::QualityRequest; +use crate::quality::syntax::{is_trivia, lower}; + +/// Significant tokens searched backwards from the column for the function name. +const MAX_FUNCTION_NAME_DISTANCE: usize = 32; +const PARSE_ONCE_FIX_REFUSAL: &str = + "Parsing the payload once requires naming a column in the CTE that reads it"; + +/// An identifier normalised the way the dialect resolves it. +type IdentifierKey = String; + +/// One JSON parse call whose argument resolves to a plain column. +struct ParseCall<'a> { + function: &'static str, + column: &'a Column, + column_start: usize, +} + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + let functions: &[&'static str] = parse_functions(request.dialect); + if functions.is_empty() { + return Vec::new(); + } + let mut found: Vec = Vec::new(); + for statement in request.statements { + for node in statement.dfs() { + if matches!(node, Expression::Select(_)) { + found.extend(scope_diagnostics( + request.tokens, + node, + functions, + request.dialect, + )); + } + } + } + found +} + +/// JSON parse functions that take one string argument, by dialect, in upper case. +fn parse_functions(dialect: Dialect) -> &'static [&'static str] { + match dialect { + Dialect::Snowflake => &["PARSE_JSON", "TRY_PARSE_JSON"], + Dialect::DuckDB => &["JSON"], + Dialect::PostgreSQL | Dialect::SQLite => &[], + } +} + +fn scope_diagnostics( + tokens: &[Token], + select: &Expression, + functions: &[&'static str], + dialect: Dialect, +) -> Vec { + let quote: char = identifier_quote(dialect); + let mut calls: Vec> = walk_in_scope(select) + .filter_map(|node| parse_call(node, functions)) + .collect(); + calls.sort_by_key(|call| call.column_start); + let mut groups: HashMap< + (&'static str, Option, IdentifierKey), + Vec<&ParseCall<'_>>, + > = HashMap::new(); + for call in &calls { + let qualifier: Option = call + .column + .table + .as_ref() + .map(|table| identifier_key(table, dialect)); + groups + .entry(( + call.function, + qualifier, + identifier_key(&call.column.name, dialect), + )) + .or_default() + .push(call); + } + let mut found: Vec = Vec::new(); + for group in groups.values().filter(|group| group.len() > 1) { + let second: &ParseCall<'_> = group[1]; + let (start, end) = function_span(tokens, second); + found.push(LintDiagnostic { + code: REPEATED_JSON_PARSE.code, + message: Cow::Borrowed(REPEATED_JSON_PARSE.message), + remediation: Cow::Owned(remediation(second, group.len(), quote)), + start, + end, + fix: None, + fix_unavailable_reason: Some(PARSE_ONCE_FIX_REFUSAL), + }); + } + found +} + +/// Normalises a name the way the dialect resolves it (Snowflake folds unquoted names upward). +fn identifier_key(identifier: &Identifier, dialect: Dialect) -> IdentifierKey { + match dialect { + Dialect::Snowflake if identifier.quoted => identifier.name.clone(), + Dialect::Snowflake => identifier.name.to_uppercase(), + _ => lower(&identifier.name), + } +} + +fn parse_call<'a>(node: &'a Expression, functions: &[&'static str]) -> Option> { + let (name, argument): (&str, &Expression) = match node { + Expression::ParseJson(function) => ("PARSE_JSON", &function.this), + Expression::Function(function) if function.args.len() == 1 => { + (function.name.as_str(), &function.args[0]) + } + _ => return None, + }; + let function: &'static str = functions + .iter() + .copied() + .find(|candidate| candidate.eq_ignore_ascii_case(name))?; + let column: &Column = unwrap_column(argument)?; + let column_start: usize = column.span.or(column.name.span)?.start; + Some(ParseCall { + function, + column, + column_start, + }) +} + +/// The column under wrappers that keep the parsed value tied to that column. +fn unwrap_column(expression: &Expression) -> Option<&Column> { + let mut current: &Expression = expression; + loop { + current = match current { + Expression::Column(column) => return Some(column), + Expression::Paren(paren) => &paren.this, + Expression::Lower(function) | Expression::Upper(function) => &function.this, + Expression::Trim(trim) if trim.characters.is_none() => &trim.this, + Expression::Cast(cast) | Expression::TryCast(cast) | Expression::SafeCast(cast) => { + &cast.this + } + Expression::Coalesce(coalesce) if coalesce.expressions.len() == 1 => { + &coalesce.expressions[0] + } + Expression::Function(function) + if function.args.len() == 1 + && call_name(current).is_some_and(|name| { + ["LOWER", "UPPER", "TRIM", "COALESCE"] + .iter() + .any(|wrapper| wrapper.eq_ignore_ascii_case(name)) + }) => + { + &function.args[0] + } + _ => return None, + }; + } +} + +/// Span of the function-name token that opens `call`, or of its column when not found. +fn function_span(tokens: &[Token], call: &ParseCall<'_>) -> (usize, usize) { + let bare: &str = call.function.rsplit('.').next().unwrap_or(call.function); + let before: usize = tokens.partition_point(|token| token.span.start < call.column_start); + tokens[..before] + .iter() + .rev() + .filter(|token| !is_trivia(token)) + .take(MAX_FUNCTION_NAME_DISTANCE) + .find(|token| token.text.eq_ignore_ascii_case(bare)) + .map_or((call.column_start, call.column_start), |token| { + (token.span.start, token.span.end) + }) +} + +fn remediation(call: &ParseCall<'_>, count: usize, quote: char) -> String { + let name: String = written(&call.column.name, quote); + let column: String = call.column.table.as_ref().map_or_else( + || name.clone(), + |table| format!("{}.{name}", written(table, quote)), + ); + let function: &str = call.function; + format!( + "{function}({column}) is parsed {count} times in one SELECT. Parse it once in the CTE that reads it, for example `{function}({name}) AS payload`, and read every field from that column in a later step." + ) +} + +/// The identifier as authored, re-quoted with the dialect's identifier quote when it was quoted. +fn written(identifier: &Identifier, quote: char) -> String { + if identifier.quoted { + let escaped: String = identifier.name.replace(quote, &format!("{quote}{quote}")); + format!("{quote}{escaped}{quote}") + } else { + identifier.name.clone() + } +} + +fn identifier_quote(_dialect: Dialect) -> char { + '"' +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/keys.rs b/crates/sqlquack-lint/src/quality/_helpers/keys.rs new file mode 100644 index 000000000..263e5adc2 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/keys.rs @@ -0,0 +1,443 @@ +//! Unique-key propagation through one model's relations, for ranking determinism proofs. + +use std::collections::HashMap; + +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::names::models::Literal; +use sqlquack_syntax::expressions::queries::models::JoinKind; +use sqlquack_syntax::expressions::queries::models::Select; + +use crate::quality::constants::FIRST_ROW; +use crate::quality::models::QualityRequest; +use crate::quality::syntax::{Projection, columns, lower, projections, qualifier, sources}; +use crate::quality::types::{InputKey, Keys}; +use crate::tree::main::is_row_number::is_row_number; + +/// Keys known for relations by name: declared upstream keys and earlier CTE outputs. +pub(crate) struct KeyEnvironment { + pub(crate) declared: HashMap, + pub(crate) ctes: HashMap, + /// Per CTE, `(row number output, partition output columns)` for deduplication filters. + pub(crate) row_numbers: HashMap)>>, +} + +impl KeyEnvironment { + /// Output keys of a query expression, by output column name, followed in a loop rather than recursion. + pub(crate) fn query_keys(&self, expression: &Expression) -> Keys { + let mut expression: &Expression = expression; + loop { + expression = match expression { + Expression::Select(select) => return self.select_keys(select), + Expression::Subquery(subquery) => &subquery.this, + Expression::Paren(paren) => &paren.this, + Expression::Union(union) if !union.all => return all_columns_key(&union.left), + Expression::Intersect(intersect) => &intersect.left, + Expression::Except(except) => &except.left, + _ => return Vec::new(), + }; + } + } + + fn relation_keys(&self, relation: &Expression) -> Keys { + match relation { + Expression::Table(table) => { + let name: String = lower(&table.name.name); + self.ctes + .get(&name) + .cloned() + .or_else(|| self.declared.get(&name).cloned()) + .unwrap_or_default() + } + Expression::Subquery(subquery) => self.query_keys(&subquery.this), + _ => Vec::new(), + } + } + + /// Keys of a select's joined and filtered input rows, before projection. + pub(crate) fn input_keys(&self, select: &Select) -> Vec { + if select + .from + .as_ref() + .is_none_or(|from| from.expressions.len() != 1) + { + return Vec::new(); + } + let found: Vec<(String, Option, &Expression)> = sources(select); + let Some((first_alias, _, first)) = found.first() else { + return Vec::new(); + }; + let mut keys: Vec = qualify(first_alias, self.relation_keys(first)); + keys.extend(qualify(first_alias, self.deduplicated_keys(select, first))); + for (join, (alias, _, relation)) in select.joins.iter().zip(found.iter().skip(1)) { + let right: Vec = qualify(alias, self.relation_keys(relation)); + let equated: Vec<(String, String)> = join_equalities(join, alias); + let left_columns: Vec<&str> = equated.iter().map(|(_, left)| left.as_str()).collect(); + let right_columns: Vec<&str> = + equated.iter().map(|(right, _)| right.as_str()).collect(); + let left_unique: bool = keys.iter().any(|key| key_within(key, &left_columns)); + let right_unique: bool = right.iter().any(|key| key_within(key, &right_columns)); + let mut joined: Vec = Vec::new(); + if right_unique + && !matches!( + join.kind, + JoinKind::Right | JoinKind::Full | JoinKind::Cross + ) + { + joined.extend(keys.iter().cloned()); + } + if left_unique && matches!(join.kind, JoinKind::Inner | JoinKind::Right) { + joined.extend(right.iter().cloned()); + } + keys = joined; + } + keys + } + + /// Input keys plus a `QUALIFY ROW_NUMBER() ... = n` partition, which holds only for output rows. + fn filtered_input_keys(&self, select: &Select) -> Vec { + let mut keys: Vec = self.input_keys(select); + if let Some(qualify_clause) = &select.qualify { + keys.extend(qualified_rank_keys(&qualify_clause.this, &sources(select))); + } + keys + } + + /// Partition keys of a `ROW_NUMBER()` output that this select filters to `= 1`. + fn deduplicated_keys(&self, select: &Select, relation: &Expression) -> Keys { + let Expression::Table(table) = relation else { + return Vec::new(); + }; + let Some(ranks) = self.row_numbers.get(&lower(&table.name.name)) else { + return Vec::new(); + }; + let filtered: Vec = select + .where_clause + .as_ref() + .map(|clause| first_row_filters(&clause.this)) + .unwrap_or_default(); + ranks + .iter() + .filter(|(rank, _)| filtered.contains(rank)) + .map(|(_, key)| key.clone()) + .collect() + } + + fn select_keys(&self, select: &Select) -> Keys { + let output: Vec> = projections(select); + let mut keys: Keys = Vec::new(); + if select.distinct + && output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + keys.push(output_names(&output)); + } + if let Some(group_by) = &select.group_by + && group_by.all.is_none() + && let Some(grain) = output_key_of(&group_by.expressions, &output) + { + keys.push(grain); + return keys; + } + let star: bool = output + .iter() + .any(|item| matches!(item, Projection::Star(_))); + for input in self.filtered_input_keys(select) { + if let Some(key) = project_key(&input, &output, star) { + keys.push(key); + } + } + keys + } +} + +fn qualify(alias: &str, keys: Keys) -> Vec { + keys.into_iter() + .map(|key| qualified_key(alias, key)) + .collect() +} + +fn qualified_key(alias: &str, key: Vec) -> InputKey { + key.into_iter() + .map(|column| (alias.to_owned(), column)) + .collect() +} + +fn key_within(key: &InputKey, columns: &[&str]) -> bool { + key.iter() + .all(|(_, column)| columns.contains(&column.as_str())) +} + +/// `(right column, left column)` pairs equated by a join's ON or USING condition. +fn join_equalities( + join: &sqlquack_syntax::expressions::queries::models::Join, + right_alias: &str, +) -> Vec<(String, String)> { + let mut pairs: Vec<(String, String)> = join + .using + .iter() + .map(|identifier| (lower(&identifier.name), lower(&identifier.name))) + .collect(); + let mut pending: Vec<&Expression> = join.on.iter().collect(); + while let Some(expression) = pending.pop() { + match expression { + Expression::And(operation) => { + pending.push(&operation.left); + pending.push(&operation.right); + } + Expression::Paren(paren) => pending.push(&paren.this), + Expression::Eq(operation) => { + if let (Expression::Column(left), Expression::Column(right)) = + (&operation.left, &operation.right) + { + let (inner, outer) = if qualifier(left).as_deref() == Some(right_alias) { + (left, right) + } else { + (right, left) + }; + pairs.push((lower(&inner.name.name), lower(&outer.name.name))); + } + } + _ => {} + } + } + pairs +} + +fn qualified_rank_keys( + condition: &Expression, + found: &[(String, Option, &Expression)], +) -> Vec { + let Expression::Eq(operation) = condition else { + return Vec::new(); + }; + let window = match (&operation.left, &operation.right) { + (Expression::WindowFunction(window), Expression::Literal(_)) + | (Expression::Literal(_), Expression::WindowFunction(window)) => window, + _ => return Vec::new(), + }; + if !is_row_number(&window.this) || window.over.partition_by.is_empty() { + return Vec::new(); + } + let mut key: InputKey = InputKey::new(); + for expression in &window.over.partition_by { + let Expression::Column(column) = expression else { + return Vec::new(); + }; + let alias: String = qualifier(column) + .or_else(|| (found.len() == 1).then(|| found[0].0.clone())) + .unwrap_or_default(); + key.insert((alias, lower(&column.name.name))); + } + vec![key] +} + +fn output_names(output: &[Projection<'_>]) -> Vec { + output + .iter() + .filter_map(|item| match item { + Projection::Named { name, .. } => Some(name.clone()), + _ => None, + }) + .collect() +} + +fn output_key_of(expressions: &[Expression], output: &[Projection<'_>]) -> Option> { + let mut key: Vec = Vec::new(); + for expression in expressions { + if let Expression::Literal(literal) = expression + && let Literal::Number(text) = literal.as_ref() + { + let Ok(ordinal) = text.parse::() else { + return None; + }; + let Some(Projection::Named { name, .. }) = ordinal + .checked_sub(1) + .and_then(|position| output.get(position)) + else { + return None; + }; + key.push(name.clone()); + continue; + } + let Expression::Column(column) = expression else { + return None; + }; + let name: String = lower(&column.name.name); + let projected: Option = output.iter().find_map(|item| match item { + Projection::Named { + name: output_name, + expression: Expression::Column(source), + } if lower(&source.name.name) == name => Some(output_name.clone()), + Projection::Named { + name: output_name, .. + } if *output_name == name => Some(output_name.clone()), + _ => None, + }); + key.push(projected?); + } + Some(key) +} + +fn project_key(input: &InputKey, output: &[Projection<'_>], star: bool) -> Option> { + let mut key: Vec = Vec::new(); + for (alias, column) in input { + let projected: Option = output.iter().find_map(|item| match item { + Projection::Named { + name, + expression: Expression::Column(source), + } if lower(&source.name.name) == *column + && qualifier(source).is_none_or(|table| table == *alias) => + { + Some(name.clone()) + } + Projection::Star(star) + if star + .table + .as_ref() + .is_none_or(|table| lower(&table.name) == *alias) => + { + Some(column.clone()) + } + _ => None, + }); + match projected { + Some(name) => key.push(name), + None if star => key.push(column.clone()), + None => return None, + } + } + Some(key) +} + +fn all_columns_key(left: &Expression) -> Keys { + let Expression::Select(select) = left else { + return Vec::new(); + }; + let output: Vec> = projections(select); + if output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + vec![output_names(&output)] + } else { + Vec::new() + } +} + +/// Columns a ranking window partitions and orders by, generously resolved to input aliases. +pub(crate) fn window_columns( + partition_by: &[Expression], + order_by: &[&Expression], +) -> Vec<(Option, String)> { + partition_by + .iter() + .chain(order_by.iter().copied()) + .flat_map(columns) + .map(|column| (qualifier(column), lower(&column.name.name))) + .collect() +} + +/// Whether some input key is covered by the window's partition and order columns. +pub(crate) fn covers(keys: &[InputKey], window: &[(Option, String)]) -> bool { + keys.iter().any(|key| key_covered(key, window)) +} + +fn key_covered(key: &InputKey, window: &[(Option, String)]) -> bool { + key.iter() + .all(|(alias, column)| window_has(window, alias, column)) +} + +fn window_has(window: &[(Option, String)], alias: &str, column: &str) -> bool { + window.iter().any(|(qualifier, name)| { + name == column && qualifier.as_deref().is_none_or(|table| table == alias) + }) +} + +/// Keys of declared relations and of each top-level CTE, in definition order. +pub(crate) fn key_environment(request: &QualityRequest<'_>) -> KeyEnvironment { + let mut environment: KeyEnvironment = KeyEnvironment { + declared: lowered_keys(request.relation_keys), + ctes: HashMap::new(), + row_numbers: HashMap::new(), + }; + for statement in request.statements { + let Expression::Select(select) = statement else { + continue; + }; + for cte in select.with.iter().flat_map(|with| with.ctes.iter()) { + let keys = environment.query_keys(&cte.this); + environment.ctes.insert(lower(&cte.alias.name), keys); + if let Expression::Select(body) = &cte.this { + environment + .row_numbers + .insert(lower(&cte.alias.name), row_number_outputs(body)); + } + } + } + environment +} + +fn lowered_keys(keys: &HashMap) -> HashMap { + let mut lowered: HashMap = HashMap::new(); + for (name, relation_keys) in keys { + let mut converted: Keys = Vec::new(); + for key in relation_keys { + let mut columns: Vec = Vec::new(); + for column in key { + columns.push(lower(column)); + } + converted.push(columns); + } + lowered.insert(lower(name), converted); + } + lowered +} + +/// `(output name, partition output columns)` for each `ROW_NUMBER() OVER (PARTITION BY ...)` output. +fn row_number_outputs(select: &Select) -> Vec<(String, Vec)> { + let output: Vec> = projections(select); + let mut found: Vec<(String, Vec)> = Vec::new(); + for item in &output { + let Projection::Named { + name, + expression: Expression::WindowFunction(window), + } = item + else { + continue; + }; + if !is_row_number(&window.this) || window.over.partition_by.is_empty() { + continue; + } + if let Some(key) = output_key_of(&window.over.partition_by, &output) { + found.push((name.clone(), key)); + } + } + found +} + +/// Columns a filter pins to the first row, as in `WHERE rn = 1 AND ...`. +fn first_row_filters(condition: &Expression) -> Vec { + let mut found: Vec = Vec::new(); + let mut pending: Vec<&Expression> = vec![condition]; + while let Some(expression) = pending.pop() { + match expression { + Expression::And(operation) => { + pending.push(&operation.left); + pending.push(&operation.right); + } + Expression::Paren(paren) => pending.push(&paren.this), + Expression::Eq(operation) => { + if let (Expression::Column(column), Expression::Literal(literal)) + | (Expression::Literal(literal), Expression::Column(column)) = + (&operation.left, &operation.right) + && matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) + { + found.push(lower(&column.name.name)); + } + } + _ => {} + } + } + found +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/literals.rs b/crates/sqlquack-lint/src/quality/_helpers/literals.rs new file mode 100644 index 000000000..ed6353e72 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/literals.rs @@ -0,0 +1,67 @@ +//! SQLCLEAN044: string literals longer than the project limit. + +use std::borrow::Cow; + +use sqlquack_syntax::tokens::models::TokenType; + +use crate::lint::models::LintDiagnostic; +use crate::quality::constants::{LONG_LITERAL, MAX_LITERAL_LENGTH_KEY, RULES_THRESHOLDS_SECTION}; +use crate::quality::models::QualityRequest; +use crate::quality::setting_help::{Setting, setting_help}; + +const CONSTANT_FIX_REFUSAL: &str = + "Moving the value to a constant requires choosing its name and placement"; + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + request + .tokens + .iter() + .filter(|token| is_string(token.token_type)) + .filter_map(|token| { + let length: usize = token.text.chars().count(); + (length > request.max_literal_length).then(|| LintDiagnostic { + code: LONG_LITERAL.code, + message: Cow::Borrowed(LONG_LITERAL.message), + remediation: Cow::Owned(remediation(length, request)), + start: token.span.start, + end: token.span.end, + fix: None, + fix_unavailable_reason: Some(CONSTANT_FIX_REFUSAL), + }) + }) + .collect() +} + +fn remediation(length: usize, request: &QualityRequest<'_>) -> String { + format!( + "{} {}", + request.long_literal_guidance, + setting_help(&Setting { + purpose: &format!( + "To allow literals of {length} characters (the current value is {})", + request.max_literal_length + ), + config_file: request.config_file, + section: RULES_THRESHOLDS_SECTION, + key: MAX_LITERAL_LENGTH_KEY, + value: &length.to_string(), + }) + ) +} + +fn is_string(kind: TokenType) -> bool { + matches!( + kind, + TokenType::String + | TokenType::DollarString + | TokenType::EscapeString + | TokenType::RawString + | TokenType::NationalString + | TokenType::UnicodeString + | TokenType::ByteString + | TokenType::TripleSingleQuotedString + | TokenType::TripleDoubleQuotedString + | TokenType::HeredocString + | TokenType::HeredocStringAlternative + ) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/projection_spans.rs b/crates/sqlquack-lint/src/quality/_helpers/projection_spans.rs new file mode 100644 index 000000000..247bab6e0 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/projection_spans.rs @@ -0,0 +1,41 @@ +//! Authored spans of the top-level items of one select list. + +use sqlquack_syntax::tokens::models::{Token, TokenType}; + +use crate::quality::constants::PROJECTION_ENDS; + +/// Spans of each top-level projection item of the select starting at token `select`. +pub(crate) fn projection_spans(tokens: &[Token], select: usize) -> Vec<(usize, usize)> { + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut depth: usize = 0; + let mut start: Option = None; + let mut end: usize = 0; + for token in tokens.iter().skip(select + 1) { + if matches!( + token.token_type, + TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment + ) { + continue; + } + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen if depth == 0 => break, + TokenType::RParen => depth -= 1, + TokenType::Comma if depth == 0 => { + if let Some(first) = start.take() { + spans.push((first, end)); + } + continue; + } + kind if depth == 0 && PROJECTION_ENDS.contains(&kind) => break, + TokenType::Distinct | TokenType::All if depth == 0 && start.is_none() => continue, + _ => {} + } + start.get_or_insert(token.span.start); + end = token.span.end; + } + if let Some(first) = start { + spans.push((first, end)); + } + spans +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/ranking.rs b/crates/sqlquack-lint/src/quality/_helpers/ranking.rs new file mode 100644 index 000000000..964eb1d72 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/ranking.rs @@ -0,0 +1,228 @@ +//! SQLCLEAN043 ranking sort caps and SQLCLEAN018 ranking determinism proofs. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use crate::tree::main::call_name::call_name; +use crate::tree::main::walk_in_scope::walk_in_scope; +use sqlquack_syntax::expressions::functions::models::WindowFunction; +use sqlquack_syntax::expressions::queries::models::Over; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::tokens::models::Token; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::lint::models::LintDiagnostic; +use crate::quality::constants::{ + MAX_RANKING_ORDER_BY_KEY, RANKING_SORT_CAP, RANKING_SORT_GUIDANCE, RULES_THRESHOLDS_SECTION, + TIE_SENSITIVE_FUNCTIONS, UNPROVEN_RANKING_MESSAGE, UNPROVEN_RANKING_REMEDIATION, + UNSTABLE_ROW_NUMBER_CODE, +}; +use crate::quality::keys::{KeyEnvironment, covers, window_columns}; +use crate::quality::models::QualityRequest; +use crate::quality::setting_help::{Setting, setting_help}; +use crate::quality::syntax::{columns, lower}; +use crate::quality::types::InputKey; + +const PRIORITY_FIX_REFUSAL: &str = + "Choosing a priority column and tie-breaker requires author intent"; +const KEY_FIX_REFUSAL: &str = "Declaring a key or adding a tie-breaker requires author intent"; + +struct RankingWindow<'a> { + function: &'static str, + window: &'a WindowFunction, + select: &'a Select, +} + +pub(crate) fn diagnostics( + request: &QualityRequest<'_>, + environment: &KeyEnvironment, +) -> Vec { + let cap: bool = request.enabled.contains(RANKING_SORT_CAP.code); + let proof: bool = request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE); + let mut used: HashSet = HashSet::new(); + let mut function_tokens: HashMap> = HashMap::new(); + for (index, token) in request.tokens.iter().enumerate() { + if TIE_SENSITIVE_FUNCTIONS + .iter() + .chain(["RANK", "DENSE_RANK"].iter()) + .any(|name| token.text.eq_ignore_ascii_case(name)) + { + function_tokens + .entry(token.text.to_ascii_uppercase()) + .or_default() + .push(index); + } + } + let mut found: Vec = Vec::new(); + for ranking in ranking_windows(request.statements) { + let over: Option<&Over> = effective_over(&ranking.window.over, ranking.select); + let order: Vec<&Expression> = over.map_or_else(Vec::new, |over| { + over.order_by.iter().map(|ordered| &ordered.this).collect() + }); + let partition: &[Expression] = over.map_or(&[], |over| over.partition_by.as_slice()); + let exceeds: bool = cap && order.len() > request.max_ranking_order_by; + let unproven: bool = proof + && !order.is_empty() + && TIE_SENSITIVE_FUNCTIONS.contains(&ranking.function) + && !proven(environment, ranking.select, partition, &order); + if !exceeds && !unproven { + continue; + } + let candidates: &[usize] = function_tokens + .get(ranking.function) + .map_or(&[], Vec::as_slice); + let token: Option = function_token( + request.tokens, + candidates, + window_anchor(partition, &order), + &used, + ); + let (start, end) = token.map_or((0, 0), |index| { + ( + request.tokens[index].span.start, + request.tokens[index].span.end, + ) + }); + used.extend(token); + if exceeds { + found.push(LintDiagnostic { + code: RANKING_SORT_CAP.code, + message: Cow::Borrowed(RANKING_SORT_CAP.message), + remediation: Cow::Owned(sort_cap_remediation( + order.len(), + request.max_ranking_order_by, + request.config_file, + )), + start, + end, + fix: None, + fix_unavailable_reason: Some(PRIORITY_FIX_REFUSAL), + }); + } + if unproven { + found.push(LintDiagnostic { + code: UNSTABLE_ROW_NUMBER_CODE, + message: Cow::Borrowed(UNPROVEN_RANKING_MESSAGE), + remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), + start, + end, + fix: None, + fix_unavailable_reason: Some(KEY_FIX_REFUSAL), + }); + } + } + found +} + +fn sort_cap_remediation(sort_keys: usize, limit: usize, config_file: &str) -> String { + format!( + "{RANKING_SORT_GUIDANCE} {}", + setting_help(&Setting { + purpose: &format!("To allow {sort_keys} sort keys (the current value is {limit})"), + config_file, + section: RULES_THRESHOLDS_SECTION, + key: MAX_RANKING_ORDER_BY_KEY, + value: &sort_keys.to_string(), + }) + ) +} + +fn proven( + environment: &KeyEnvironment, + select: &Select, + partition: &[Expression], + order: &[&Expression], +) -> bool { + let keys: Vec = environment.input_keys(select); + covers(&keys, &window_columns(partition, order)) +} + +fn ranking_windows(statements: &[Expression]) -> Vec> { + let mut found: Vec> = Vec::new(); + for statement in statements { + for node in statement.dfs() { + let Expression::Select(select) = node else { + continue; + }; + for local in walk_in_scope(node) { + let Expression::WindowFunction(window) = local else { + continue; + }; + let function: &'static str = match &window.this { + Expression::RowNumber(_) => "ROW_NUMBER", + Expression::Rank(_) => "RANK", + Expression::DenseRank(_) => "DENSE_RANK", + Expression::FirstValue(_) => "FIRST_VALUE", + Expression::LastValue(_) => "LAST_VALUE", + written => match call_name(written).map(str::to_ascii_uppercase).as_deref() { + Some("ROW_NUMBER") => "ROW_NUMBER", + Some("RANK") => "RANK", + Some("DENSE_RANK") => "DENSE_RANK", + Some("FIRST_VALUE") => "FIRST_VALUE", + Some("LAST_VALUE") => "LAST_VALUE", + _ => continue, + }, + }; + found.push(RankingWindow { + function, + window, + select, + }); + } + } + } + found +} + +fn effective_over<'a>(over: &'a Over, select: &'a Select) -> Option<&'a Over> { + let mut current: &Over = over; + let mut seen: HashSet = HashSet::new(); + loop { + if !current.order_by.is_empty() || current.window_name.is_none() { + return Some(current); + } + let name: String = lower(¤t.window_name.as_ref()?.name); + if !seen.insert(name.clone()) { + return None; + } + current = &select + .windows + .iter() + .flatten() + .find(|window| lower(&window.name.name) == name)? + .spec; + } +} + +/// Start offset of the first located column a window partitions or orders by. +fn window_anchor(partition: &[Expression], order: &[&Expression]) -> Option { + let mut expressions: Vec<&Expression> = partition.iter().collect(); + expressions.extend(order.iter().copied()); + for expression in expressions { + for column in columns(expression) { + if let Some(span) = column.span { + return Some(span.start); + } + } + } + None +} + +/// The function-name token before the anchor, or the first token not reported yet. +fn function_token( + tokens: &[Token], + candidates: &[usize], + anchor: Option, + used: &HashSet, +) -> Option { + if let Some(anchor) = anchor { + let before: usize = candidates.partition_point(|&index| tokens[index].span.start < anchor); + if let Some(position) = before.checked_sub(1) { + return Some(candidates[position]); + } + } + candidates + .iter() + .copied() + .find(|index| !used.contains(index)) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/removal_safety.rs b/crates/sqlquack-lint/src/quality/_helpers/removal_safety.rs new file mode 100644 index 000000000..2675fc601 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/removal_safety.rs @@ -0,0 +1,111 @@ +//! SQLCLEAN042 allow-list: when removing an unused CTE output provably keeps every row. + +use crate::tree::main::contains_aggregate::contains_aggregate; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::quality::constants::{ + DETERMINISTIC_SCALAR_FUNCTIONS, MIN_REMOVABLE_OUTPUTS, MULTI_ARGUMENT_SCALAR_FUNCTIONS, + ROW_NEUTRAL_EXPRESSIONS, SINGLE_ARGUMENT_SCALAR_FUNCTIONS, +}; + +const ORDER_BY_ALL: &str = "all"; +const DISTINCT_REFUSAL: &str = "SELECT DISTINCT depends on every output column"; +const DISTINCT_ON_REFUSAL: &str = "SELECT DISTINCT ON keeps one row per key chosen by the select"; +const GROUP_BY_ALL_REFUSAL: &str = "GROUP BY ALL groups by every non-aggregate output column"; +const ORDER_BY_ALL_REFUSAL: &str = "ORDER BY ALL sorts by every output column"; +const UNGROUPED_AGGREGATE_REFUSAL: &str = + "An aggregate select without GROUP BY depends on its whole projection"; +const POSITIONAL_REFUSAL: &str = "Positional GROUP BY or ORDER BY would shift"; +const ONLY_OUTPUT_REFUSAL: &str = "Removing the only output column leaves an empty select"; +const EXPRESSION_REFUSAL: &str = "Only column references and deterministic scalar expressions are removed automatically; this expression may change the rows the CTE returns"; + +/// Why removing any output of `select` could change its rows, if it could. +pub(super) fn select_refusal(select: &Select, outputs: usize) -> Option<&'static str> { + if select.distinct { + Some(DISTINCT_REFUSAL) + } else if select.distinct_on.is_some() { + Some(DISTINCT_ON_REFUSAL) + } else if select + .group_by + .as_ref() + .is_some_and(|group| group.all == Some(true)) + { + Some(GROUP_BY_ALL_REFUSAL) + } else if orders_by_all(select) { + Some(ORDER_BY_ALL_REFUSAL) + } else if select.group_by.is_none() + && (select.having.is_some() || select.expressions.iter().any(contains_aggregate)) + { + Some(UNGROUPED_AGGREGATE_REFUSAL) + } else if positional_clauses(select) { + Some(POSITIONAL_REFUSAL) + } else if outputs < MIN_REMOVABLE_OUTPUTS { + Some(ONLY_OUTPUT_REFUSAL) + } else { + None + } +} + +/// Why removing this one projection could change the rows, if it could. +pub(super) fn expression_refusal(expression: &Expression) -> Option<&'static str> { + (!row_neutral(expression)).then_some(EXPRESSION_REFUSAL) +} + +/// Whether a star select depends on every input column or on their positions. +pub(super) fn reads_whole_rows(select: &Select) -> bool { + select.distinct + || select.distinct_on.is_some() + || select.group_by.is_some() + || orders_by_all(select) + || positional_clauses(select) +} + +/// A column reference or deterministic scalar expression (no aggregate, window, SRF or subquery). +fn row_neutral(expression: &Expression) -> bool { + expression.dfs().all(|node| match node { + Expression::Function(function) => { + let name: String = function.name.to_ascii_uppercase(); + let arguments: usize = function.args.len(); + !function.distinct + && DETERMINISTIC_SCALAR_FUNCTIONS.contains(&name.as_str()) + && (arguments == 1 || !SINGLE_ARGUMENT_SCALAR_FUNCTIONS.contains(&name.as_str())) + && (arguments > 1 || !MULTI_ARGUMENT_SCALAR_FUNCTIONS.contains(&name.as_str())) + } + other => ROW_NEUTRAL_EXPRESSIONS.contains(&other.variant_name()), + }) +} + +fn orders_by_all(select: &Select) -> bool { + select + .order_by + .as_ref() + .is_some_and(|order| order.all.is_some()) + || select + .order_by + .iter() + .flat_map(|order| order.expressions.iter()) + .any(|ordered| match &ordered.this { + Expression::Column(column) => { + column.table.is_none() + && !column.name.quoted + && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL) + } + Expression::Var(var) => var.this.eq_ignore_ascii_case(ORDER_BY_ALL), + _ => false, + }) +} + +fn positional_clauses(select: &Select) -> bool { + let numeric = |expression: &Expression| matches!(expression, Expression::Literal(_)); + select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()) + .any(numeric) + || select + .order_by + .iter() + .flat_map(|order| order.expressions.iter()) + .any(|ordered| numeric(&ordered.this)) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/safety.rs b/crates/sqlquack-lint/src/quality/_helpers/safety.rs new file mode 100644 index 000000000..ca9238705 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/safety.rs @@ -0,0 +1,716 @@ +//! SQLCLEAN046 to SQLCLEAN053: report-only safety Rules for queries that cannot do what they appear to. + +use std::borrow::Cow; +use std::collections::HashSet; + +use crate::tree::main::is_aggregate::is_aggregate; +use sqlquack_syntax::expressions::names::models::Identifier; +use sqlquack_syntax::expressions::queries::models::Join; +use sqlquack_syntax::expressions::queries::models::With; +use sqlquack_syntax::expressions::statements::models::Delete; +use sqlquack_syntax::expressions::statements::models::Insert; +use sqlquack_syntax::expressions::statements::models::Update; +use sqlquack_syntax::tokens::models::{Token, TokenType}; +use sqlquack_syntax::{Dialect, Expression, ExpressionWalk}; + +use crate::lint::models::{LintDiagnostic, LintRuleMetadata}; +use crate::quality::constants::{ + DUPLICATE_CTE_NAME, INSERT_WITHOUT_COLUMNS, LITERAL_ZERO_DIVISION, MISPLACED_AGGREGATE, + NATURAL_JOIN, NON_SNOWFLAKE_AGGREGATE_CALLS, NULL_MATCH, REVERSED_BETWEEN, UNFILTERED_WRITE, +}; +use crate::quality::models::QualityRequest; +use crate::quality::syntax::is_trivia; + +const RENAME_REFUSAL: &str = "Renaming a CTE requires choosing which definition its readers mean"; +const NULL_TEST_REFUSAL: &str = "Rewriting to IS NULL changes what the query returns"; +const DIVISOR_REFUSAL: &str = "Choosing a divisor or a NULL guard requires the intended result"; +const WRITE_SCOPE_REFUSAL: &str = "Only the author knows which rows the statement should change"; +const COLUMN_LIST_REFUSAL: &str = "Listing the target columns requires the table's schema"; +const JOIN_CONDITION_REFUSAL: &str = "Writing the join condition requires both relations' schemas"; +const BOUNDS_REFUSAL: &str = "Swapping the bounds changes what the query returns"; +const AGGREGATE_REFUSAL: &str = + "Moving the expression to HAVING, QUALIFY or an outer query requires restructuring the query"; +const ASYMMETRIC: &str = "ASYMMETRIC"; +const MODULO_FUNCTION: &str = "MOD"; +const SINGLE_ARGUMENT_AGGREGATES: [&str; 2] = ["MIN", "MAX"]; +const ISO_DATE_LENGTH: usize = 10; + +/// A finding's start and end character offsets. +type CharSpan = (usize, usize); + +/// Significant tokens with their parenthesis depth. +struct Significant<'a> { + tokens: Vec<&'a Token>, + depths: Vec, +} + +impl<'a> Significant<'a> { + fn new(tokens: &'a [Token]) -> Self { + let tokens: Vec<&Token> = tokens.iter().filter(|token| !is_trivia(token)).collect(); + let mut depths: Vec = Vec::with_capacity(tokens.len()); + let mut depth: usize = 0; + for token in &tokens { + if token.token_type == TokenType::RParen { + depth = depth.saturating_sub(1); + } + depths.push(depth); + if token.token_type == TokenType::LParen { + depth += 1; + } + } + Self { tokens, depths } + } + + fn kind(&self, index: usize) -> Option { + self.tokens.get(index).map(|token| token.token_type) + } + + fn is(&self, index: usize, kind: TokenType) -> bool { + self.kind(index) == Some(kind) + } + + fn text_is(&self, index: usize, text: &str) -> bool { + self.tokens + .get(index) + .is_some_and(|token| token.text.eq_ignore_ascii_case(text)) + } + + fn close(&self, open: usize) -> Option { + (open + 1..self.tokens.len()).find(|&index| { + self.is(index, TokenType::RParen) && self.depths[index] == self.depths[open] + }) + } + + fn span(&self, first: usize, last: usize) -> CharSpan { + (self.tokens[first].span.start, self.tokens[last].span.end) + } + + /// Top-level comma positions inside the parentheses that open at `open`. + fn arguments_commas(&self, open: usize, close: usize) -> Vec { + (open + 1..close) + .filter(|&index| { + self.is(index, TokenType::Comma) && self.depths[index] == self.depths[open] + 1 + }) + .collect() + } + + /// Non-empty statements split at top-level semicolons, as index ranges. + fn statements(&self) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut start: usize = 0; + for index in 0..=self.tokens.len() { + let ends = index == self.tokens.len() + || (self.is(index, TokenType::Semicolon) && self.depths[index] == 0); + if ends { + if index > start { + ranges.push((start, index)); + } + start = index + 1; + } + } + ranges + } +} + +/// A literal bound whose order is exact: a number, or an ISO `DATE` literal. +#[derive(PartialEq, PartialOrd)] +enum Bound { + Number(f64), + Date(String), +} + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + let significant = Significant::new(request.tokens); + let enabled = |rule: &LintRuleMetadata| request.enabled.contains(rule.code); + let mut found: Vec = Vec::new(); + let mut report = + |rule: &'static LintRuleMetadata, reason: &'static str, spans: Vec| { + found.extend(spans.into_iter().map(|(start, end)| LintDiagnostic { + code: rule.code, + message: Cow::Borrowed(rule.message), + remediation: Cow::Borrowed(rule.remediation), + start, + end, + fix: None, + fix_unavailable_reason: Some(reason), + })); + }; + if enabled(&DUPLICATE_CTE_NAME) { + report( + &DUPLICATE_CTE_NAME, + RENAME_REFUSAL, + duplicate_cte_spans(request.statements, request.dialect), + ); + } + if enabled(&NULL_MATCH) { + report( + &NULL_MATCH, + NULL_TEST_REFUSAL, + null_match_spans(&significant), + ); + } + if enabled(&LITERAL_ZERO_DIVISION) { + report( + &LITERAL_ZERO_DIVISION, + DIVISOR_REFUSAL, + zero_division_spans(&significant), + ); + } + if enabled(&UNFILTERED_WRITE) || enabled(&INSERT_WITHOUT_COLUMNS) { + let (writes, inserts) = statement_spans(&significant, request.statements); + if enabled(&UNFILTERED_WRITE) { + report(&UNFILTERED_WRITE, WRITE_SCOPE_REFUSAL, writes); + } + if enabled(&INSERT_WITHOUT_COLUMNS) { + report(&INSERT_WITHOUT_COLUMNS, COLUMN_LIST_REFUSAL, inserts); + } + } + if enabled(&NATURAL_JOIN) { + report( + &NATURAL_JOIN, + JOIN_CONDITION_REFUSAL, + natural_join_spans(&significant), + ); + } + if enabled(&REVERSED_BETWEEN) { + report( + &REVERSED_BETWEEN, + BOUNDS_REFUSAL, + reversed_between_spans(&significant), + ); + } + if enabled(&MISPLACED_AGGREGATE) { + report( + &MISPLACED_AGGREGATE, + AGGREGATE_REFUSAL, + misplaced_aggregate_spans( + &significant, + &aggregate_names(request.statements, request.dialect), + ), + ); + } + found +} + +fn duplicate_cte_spans(statements: &[Expression], dialect: Dialect) -> Vec { + let mut spans: Vec = Vec::new(); + for statement in statements { + for with in statement.dfs().filter_map(with_clause) { + let mut seen: HashSet = HashSet::new(); + for cte in &with.ctes { + if !seen.insert(cte_key(&cte.alias, dialect)) + && let Some(span) = cte.alias.span + { + spans.push((span.start, span.end)); + } + } + } + } + spans +} + +fn with_clause(node: &Expression) -> Option<&With> { + match node { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(operation) => operation.with.as_ref(), + Expression::Intersect(operation) => operation.with.as_ref(), + Expression::Except(operation) => operation.with.as_ref(), + Expression::Insert(insert) => insert.with.as_ref(), + Expression::Update(update) => update.with.as_ref(), + Expression::Delete(delete) => delete.with.as_ref(), + _ => None, + } +} + +/// A CTE name as the dialect resolves it, so only names the engine treats as equal collide. +fn cte_key(identifier: &Identifier, dialect: Dialect) -> String { + let quoted_names_ignore_case = matches!(dialect, Dialect::DuckDB | Dialect::SQLite); + match dialect { + _ if identifier.quoted && !quoted_names_ignore_case => identifier.name.clone(), + Dialect::Snowflake => identifier.name.to_uppercase(), + _ => identifier.name.to_lowercase(), + } +} + +/// `CASE x WHEN NULL` and a singleton positive `IN (NULL)`; SQLCLEAN019 owns `NOT IN`. +fn null_match_spans(significant: &Significant<'_>) -> Vec { + let mut spans: Vec = Vec::new(); + for index in 0..significant.tokens.len() { + match significant.kind(index) { + Some(TokenType::In) + if !(index > 0 && significant.is(index - 1, TokenType::Not)) + && significant.is(index + 1, TokenType::LParen) + && significant.is(index + 2, TokenType::Null) + && significant.is(index + 3, TokenType::RParen) => + { + spans.push(significant.span(index, index + 3)); + } + Some(TokenType::Case) if !significant.is(index + 1, TokenType::When) => { + spans.extend(simple_case_null_spans(significant, index)); + } + _ => {} + } + } + spans +} + +fn simple_case_null_spans(significant: &Significant<'_>, case: usize) -> Vec { + let depth: usize = significant.depths[case]; + let mut nested: usize = 0; + let mut spans: Vec = Vec::new(); + for index in case + 1..significant.tokens.len() { + if significant.depths[index] < depth { + break; + } + if significant.depths[index] > depth { + continue; + } + match significant.kind(index) { + Some(TokenType::Case) => nested += 1, + Some(TokenType::End) if nested == 0 => break, + Some(TokenType::End) => nested -= 1, + Some(TokenType::When) + if nested == 0 + && significant.is(index + 1, TokenType::Null) + && significant.is(index + 2, TokenType::Then) => + { + spans.push(significant.span(index, index + 1)); + } + _ => {} + } + } + spans +} + +fn zero_division_spans(significant: &Significant<'_>) -> Vec { + let mut spans: Vec = Vec::new(); + for index in 0..significant.tokens.len() { + let operator = matches!( + significant.kind(index), + Some(TokenType::Slash | TokenType::Percent | TokenType::Div) + ); + if operator + && let Some((number, end)) = numeric_operand(significant, index + 1) + && is_zero(&significant.tokens[number].text) + && !binds_tighter(significant.kind(end + 1)) + { + spans.push(significant.span(index, end)); + } + if significant.text_is(index, MODULO_FUNCTION) + && significant.is(index + 1, TokenType::LParen) + && let Some(close) = significant.close(index + 1) + && let [comma] = significant.arguments_commas(index + 1, close)[..] + && let Some((number, end)) = numeric_operand(significant, comma + 1) + && is_zero(&significant.tokens[number].text) + && end + 1 == close + { + spans.push(significant.span(index, close)); + } + } + spans +} + +/// Operators that bind tighter than division, so a following one changes the divisor. +fn binds_tighter(kind: Option) -> bool { + matches!( + kind, + Some( + TokenType::DColon + | TokenType::DStar + | TokenType::Caret + | TokenType::LBracket + | TokenType::Dot + ) + ) +} + +/// Whether a number literal's digits are all zero, whatever its exponent. +fn is_zero(text: &str) -> bool { + let mantissa: &str = text.split(['e', 'E']).next().unwrap_or(text); + mantissa.chars().any(|character| character.is_ascii_digit()) + && mantissa + .chars() + .all(|character| matches!(character, '0' | '.' | '_')) +} + +/// An optionally signed and parenthesised number literal at `start`: its number token and last index. +fn numeric_operand(significant: &Significant<'_>, start: usize) -> Option<(usize, usize)> { + let mut index: usize = start; + let mut opens: usize = 0; + while significant.is(index, TokenType::LParen) { + opens += 1; + index += 1; + } + if matches!( + significant.kind(index), + Some(TokenType::Dash | TokenType::Plus) + ) { + index += 1; + } + if !significant.is(index, TokenType::Number) { + return None; + } + let mut end: usize = index; + for _ in 0..opens { + if !significant.is(end + 1, TokenType::RParen) { + return None; + } + end += 1; + } + Some((index, end)) +} + +/// A number or an ISO `DATE 'YYYY-MM-DD'` literal starting at `start`, and its last index. +fn bound(significant: &Significant<'_>, start: usize) -> Option<(Bound, usize)> { + if significant.is(start, TokenType::Date) && significant.is(start + 1, TokenType::String) { + let text: &str = &significant.tokens[start + 1].text; + let iso: bool = text.len() == ISO_DATE_LENGTH + && text + .char_indices() + .all(|(position, character)| match position { + 4 | 7 => character == '-', + _ => character.is_ascii_digit(), + }); + return iso.then(|| (Bound::Date(text.to_owned()), start + 1)); + } + let (number, end) = numeric_operand(significant, start)?; + let Ok(magnitude) = significant.tokens[number] + .text + .replace('_', "") + .parse::() + else { + return None; + }; + let negative: bool = significant.is(number - 1, TokenType::Dash); + Some(( + Bound::Number(if negative { -magnitude } else { magnitude }), + end, + )) +} + +fn reversed_between_spans(significant: &Significant<'_>) -> Vec { + let mut spans: Vec = Vec::new(); + for index in 0..significant.tokens.len() { + if !significant.is(index, TokenType::Between) { + continue; + } + let low_start: usize = index + 1 + usize::from(significant.text_is(index + 1, ASYMMETRIC)); + if let Some((low, low_end)) = bound(significant, low_start) + && significant.is(low_end + 1, TokenType::And) + && let Some((high, high_end)) = bound(significant, low_end + 2) + && ends_bound(significant.kind(high_end + 1)) + && matches!( + (&low, &high), + (Bound::Number(_), Bound::Number(_)) | (Bound::Date(_), Bound::Date(_)) + ) + && low > high + { + spans.push(significant.span(index, high_end)); + } + } + spans +} + +/// Tokens that bind looser than BETWEEN, so the upper bound ends before them. +fn ends_bound(kind: Option) -> bool { + match kind { + None => true, + Some(kind) => matches!( + kind, + TokenType::RParen + | TokenType::Comma + | TokenType::Semicolon + | TokenType::And + | TokenType::Or + | TokenType::Then + | TokenType::When + | TokenType::Else + | TokenType::End + | TokenType::As + | TokenType::From + | TokenType::Where + | TokenType::Group + | TokenType::Having + | TokenType::Qualify + | TokenType::Window + | TokenType::Order + | TokenType::Limit + | TokenType::Union + | TokenType::Except + | TokenType::Intersect + ), + } +} + +/// UPDATE or DELETE without a limiting predicate, and INSERT without a column list. +fn statement_spans( + significant: &Significant<'_>, + statements: &[Expression], +) -> (Vec, Vec) { + let mut writes: Vec = Vec::new(); + let mut inserts: Vec = Vec::new(); + let ranges: Vec<(usize, usize)> = significant.statements(); + if ranges.len() != statements.len() { + return (writes, inserts); + } + for (statement, (start, end)) in statements.iter().zip(ranges) { + let keyword: usize = (start..end) + .find(|&index| { + significant.depths[index] == 0 + && matches!( + significant.kind(index), + Some(TokenType::Update | TokenType::Delete | TokenType::Insert) + ) + }) + .unwrap_or(start); + let span = significant.span(keyword, keyword); + match statement { + Expression::Update(update) if !update_is_limited(update) => writes.push(span), + Expression::Delete(delete) if !delete_is_limited(delete) => writes.push(span), + Expression::Insert(insert) if lacks_column_list(insert) => inserts.push(span), + _ => {} + } + } + (writes, inserts) +} + +fn update_is_limited(update: &Update) -> bool { + update + .where_clause + .as_ref() + .is_some_and(|clause| reads_rows(&clause.this)) + || update.limit.is_some() + || update + .table_joins + .iter() + .chain(&update.from_joins) + .any(join_is_limiting) +} + +fn delete_is_limited(delete: &Delete) -> bool { + delete + .where_clause + .as_ref() + .is_some_and(|clause| reads_rows(&clause.this)) + || delete.limit.is_some() + || delete.joins.iter().any(join_is_limiting) +} + +/// Whether a predicate depends on the rows: it reads a column or a subquery, unlike `WHERE TRUE`. +fn reads_rows(predicate: &Expression) -> bool { + predicate.dfs().any(|node| { + matches!( + node, + Expression::Column(_) | Expression::Select(_) | Expression::Subquery(_) + ) + }) +} + +fn join_is_limiting(join: &Join) -> bool { + join.on.as_ref().is_some_and(reads_rows) || !join.using.is_empty() +} + +fn lacks_column_list(insert: &Insert) -> bool { + insert.columns.is_empty() + && !insert.default_values + && !insert.by_name + && insert.directory.is_none() + && (insert.query.is_some() || !insert.values.is_empty()) +} + +fn natural_join_spans(significant: &Significant<'_>) -> Vec { + let mut spans: Vec = Vec::new(); + for index in 0..significant.tokens.len() { + if !significant.is(index, TokenType::Natural) { + continue; + } + let mut join: usize = index + 1; + while matches!( + significant.kind(join), + Some( + TokenType::Left + | TokenType::Right + | TokenType::Full + | TokenType::Inner + | TokenType::Outer + ) + ) { + join += 1; + } + if significant.is(join, TokenType::Join) { + spans.push(significant.span(index, join)); + } + } + spans +} + +/// Function names the parser classified as aggregates, upper-cased without underscores. +fn aggregate_names(statements: &[Expression], dialect: Dialect) -> HashSet { + statements + .iter() + .flat_map(|statement| statement.dfs()) + .filter(|node| is_aggregate(node)) + .filter_map(|node| match node { + Expression::AggregateFunction(function) + if dialect == Dialect::Snowflake && is_non_snowflake_aggregate(&function.name) => + { + None + } + Expression::AggregateFunction(function) => Some(normalized_name(&function.name)), + other => Some(normalized_name(other.variant_name())), + }) + .collect() +} + +/// Whether `name` is another dialect's aggregate that Snowflake does not have. +fn is_non_snowflake_aggregate(name: &str) -> bool { + NON_SNOWFLAKE_AGGREGATE_CALLS + .iter() + .any(|aggregate| aggregate.eq_ignore_ascii_case(name)) +} + +fn normalized_name(name: &str) -> String { + name.chars() + .filter(|character| *character != '_') + .flat_map(char::to_uppercase) + .collect() +} + +/// Aggregate and window calls directly in WHERE, ON or GROUP BY, outside nested queries. +fn misplaced_aggregate_spans( + significant: &Significant<'_>, + aggregates: &HashSet, +) -> Vec { + let mut spans: Vec = Vec::new(); + for index in 0..significant.tokens.len() { + let first: usize = match significant.kind(index) { + Some(TokenType::Where | TokenType::On) => index + 1, + Some(TokenType::Group) if significant.is(index + 1, TokenType::By) => index + 2, + _ => continue, + }; + spans.extend(clause_call_spans( + significant, + (first, significant.depths[index]), + significant.is(index, TokenType::On), + aggregates, + )); + } + spans +} + +fn clause_call_spans( + significant: &Significant<'_>, + (first, depth): (usize, usize), + comma_ends: bool, + aggregates: &HashSet, +) -> Vec { + let mut spans: Vec = Vec::new(); + let mut index: usize = first; + while index < significant.tokens.len() { + if significant.depths[index] < depth + || (significant.depths[index] == depth && ends_clause(significant, index, comma_ends)) + { + break; + } + if significant.is(index, TokenType::LParen) + && matches!( + significant.kind(index + 1), + Some(TokenType::Select | TokenType::With | TokenType::Values) + ) + { + index = significant + .close(index) + .map_or(significant.tokens.len(), |close| close + 1); + continue; + } + if let Some((close, end)) = call(significant, index) { + if significant.is(close + 1, TokenType::Over) { + spans.push(significant.span(index, end)); + index = end + 1; + continue; + } + if is_aggregate_call(significant, index, close, aggregates) { + spans.push(significant.span(index, close)); + index = close + 1; + continue; + } + } + index += 1; + } + spans +} + +/// A function call at `name`: its closing parenthesis and the end of any OVER specification. +fn call(significant: &Significant<'_>, name: usize) -> Option<(usize, usize)> { + let text: &str = &significant.tokens[name].text; + let is_name: bool = text + .chars() + .next() + .is_some_and(|character| character.is_ascii_alphabetic() || character == '_') + && text + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_'); + if !is_name + || !significant.is(name + 1, TokenType::LParen) + || (name > 0 && significant.is(name - 1, TokenType::Dot)) + { + return None; + } + let close: usize = significant.close(name + 1)?; + let end: usize = if significant.is(close + 1, TokenType::Over) { + if significant.is(close + 2, TokenType::LParen) { + significant.close(close + 2)? + } else { + close + 2 + } + } else { + close + }; + (end < significant.tokens.len()).then_some((close, end)) +} + +fn is_aggregate_call( + significant: &Significant<'_>, + name: usize, + close: usize, + aggregates: &HashSet, +) -> bool { + let normalized: String = normalized_name(&significant.tokens[name].text); + aggregates.contains(&normalized) + && (!SINGLE_ARGUMENT_AGGREGATES.contains(&normalized.as_str()) + || significant.arguments_commas(name + 1, close).is_empty()) +} + +/// Whether `index` ends the clause; only an ON condition ends at a comma, before the next relation. +fn ends_clause(significant: &Significant<'_>, index: usize, comma_ends: bool) -> bool { + match significant.kind(index) { + Some(TokenType::Group) => significant.is(index + 1, TokenType::By), + Some(TokenType::Comma) => comma_ends, + Some(kind) => matches!( + kind, + TokenType::Where + | TokenType::Having + | TokenType::Qualify + | TokenType::Window + | TokenType::Order + | TokenType::Limit + | TokenType::Union + | TokenType::Except + | TokenType::Intersect + | TokenType::Returning + | TokenType::Fetch + | TokenType::Offset + | TokenType::Semicolon + | TokenType::Join + | TokenType::Inner + | TokenType::Left + | TokenType::Right + | TokenType::Full + | TokenType::Cross + | TokenType::Natural + | TokenType::Set + | TokenType::Using + ), + None => true, + } +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/setting_help.rs b/crates/sqlquack-lint/src/quality/_helpers/setting_help.rs new file mode 100644 index 000000000..a456ab7f6 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/setting_help.rs @@ -0,0 +1,26 @@ +//! Remediation text that shows the exact configuration setting to write. + +use crate::constants::SETTING_SNIPPET_INDENT; + +/// One setting a remediation tells the user to write. +pub(crate) struct Setting<'a> { + pub(crate) purpose: &'a str, + pub(crate) config_file: &'a str, + pub(crate) section: &'a str, + pub(crate) key: &'a str, + pub(crate) value: &'a str, +} + +/// `, set this in :` followed by the section and `key = value`. +pub(crate) fn setting_help(setting: &Setting<'_>) -> String { + let Setting { + purpose, + config_file, + section, + key, + value, + } = setting; + format!( + "{purpose}, set this in {config_file}:\n{SETTING_SNIPPET_INDENT}[{section}]\n{SETTING_SNIPPET_INDENT}{key} = {value}" + ) +} diff --git a/crates/sqlquack-lint/src/quality/_helpers/syntax.rs b/crates/sqlquack-lint/src/quality/_helpers/syntax.rs new file mode 100644 index 000000000..d05e26abc --- /dev/null +++ b/crates/sqlquack-lint/src/quality/_helpers/syntax.rs @@ -0,0 +1,126 @@ +//! Shared syntax helpers for SQL quality Rules. + +use sqlquack_syntax::expressions::names::models::Column; +use sqlquack_syntax::expressions::names::models::Star; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::tokens::models::{Token, TokenType}; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::quality::constants::MIN_REMOVABLE_OUTPUTS; +use rustc_hash::FxHashMap; + +/// One projection of a select list. +pub(super) enum Projection<'a> { + Named { + name: String, + expression: &'a Expression, + }, + Star(&'a Star), + Unnamed, +} + +pub(super) fn lower(name: &str) -> String { + name.to_ascii_lowercase() +} + +pub(super) fn projections(select: &Select) -> Vec> { + select + .expressions + .iter() + .map(|expression| match expression { + Expression::Alias(alias) => Projection::Named { + name: lower(&alias.alias.name), + expression: &alias.this, + }, + Expression::Column(column) => Projection::Named { + name: lower(&column.name.name), + expression, + }, + Expression::Star(star) => Projection::Star(star), + _ => Projection::Unnamed, + }) + .collect() +} + +pub(super) fn columns(expression: &Expression) -> Vec<&Column> { + expression + .dfs() + .filter_map(|node| match node { + Expression::Column(column) => Some(column.as_ref()), + _ => None, + }) + .collect() +} + +pub(super) fn qualifier(column: &Column) -> Option { + column.table.as_ref().map(|table| lower(&table.name)) +} + +/// Relations read directly by one select: `(alias, relation name, expression)`. +pub(super) fn sources(select: &Select) -> Vec<(String, Option, &Expression)> { + let mut found: Vec<(String, Option, &Expression)> = Vec::new(); + let from = select.from.iter().flat_map(|from| from.expressions.iter()); + for expression in from.chain(select.joins.iter().map(|join| &join.this)) { + match expression { + Expression::Table(table) => { + let name: String = lower(&table.name.name); + let alias: String = table + .alias + .as_ref() + .map_or_else(|| name.clone(), |alias| lower(&alias.name)); + found.push((alias, Some(name), expression)); + } + Expression::Subquery(subquery) => { + let alias: String = subquery + .alias + .as_ref() + .map_or_else(String::new, |alias| lower(&alias.name)); + found.push((alias, None, expression)); + } + _ => found.push((String::new(), None, expression)), + } + } + found +} + +/// Token index of the SELECT opening each `name AS (SELECT` CTE body, by lower-cased name. +pub(super) fn cte_select_tokens(tokens: &[Token]) -> FxHashMap { + let significant: Vec = (0..tokens.len()) + .filter(|&index| !is_trivia(&tokens[index])) + .collect(); + let mut found: FxHashMap = FxHashMap::default(); + for window in significant.windows(4) { + let [name_index, as_index, paren_index, select_index] = window else { + continue; + }; + if tokens[*as_index].token_type == TokenType::As + && tokens[*paren_index].token_type == TokenType::LParen + && tokens[*select_index].token_type == TokenType::Select + { + found + .entry(lower(&tokens[*name_index].text)) + .or_insert(*select_index); + } + } + found +} + +/// The comma-inclusive deletion span that removes projection `index` from `spans`. +pub(super) fn projection_removal(spans: &[(usize, usize)], index: usize) -> Option<(usize, usize)> { + if spans.len() < MIN_REMOVABLE_OUTPUTS { + return None; + } + let (start, end) = spans[index]; + if index + 1 < spans.len() { + Some((start, spans[index + 1].0)) + } else { + Some((spans[index - 1].1, end)) + } +} + +pub(super) fn is_trivia(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment + ) +} diff --git a/crates/sqlquack-lint/src/quality/constants.rs b/crates/sqlquack-lint/src/quality/constants.rs new file mode 100644 index 000000000..cea3529a5 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/constants.rs @@ -0,0 +1,272 @@ +use crate::lint::models::LintRuleMetadata; +use sqlquack_syntax::tokens::models::TokenType; + +pub(crate) const UNUSED_CTE_OUTPUT: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN042", + message: "CTE output column is never read later in the query", + remediation: "Remove the column from the CTE output, or read it in a later CTE or the final query.", +}; +pub(crate) const RANKING_SORT_CAP: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN043", + message: "Ranking window sorts by more expressions than max_ranking_order_by", + remediation: "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker.", +}; +pub(crate) const LONG_LITERAL: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN044", + message: "String literal is longer than max_literal_length", + remediation: "Shorten the literal, or move the value into a reference table or a query parameter.", +}; +pub(crate) const REPEATED_JSON_PARSE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN045", + message: "JSON payload is parsed more than once in one SELECT", + remediation: "Parse the payload once in the CTE that reads it, for example `TRY_PARSE_JSON(value) AS payload`, and read every field from that column in a later step.", +}; +pub(crate) const DUPLICATE_CTE_NAME: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN046", + message: "WITH defines the same CTE name more than once", + remediation: "Give each CTE in one WITH clause its own name; engines reject the duplicate or silently read only one definition.", +}; +pub(crate) const NULL_MATCH: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN047", + message: "CASE WHEN NULL or IN (NULL) never matches", + remediation: "A simple CASE compares with =, and x IN (NULL) means x = NULL, so neither is ever true. Test the value with IS NULL, for example CASE WHEN x IS NULL THEN ... END.", +}; +pub(crate) const LITERAL_ZERO_DIVISION: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN048", + message: "Expression divides by a literal zero", + remediation: "Depending on the dialect this raises an error or returns NULL. Remove the division, or divide by NULLIF(divisor, 0) when the divisor can legitimately be zero.", +}; +pub(crate) const UNFILTERED_WRITE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN049", + message: "UPDATE or DELETE has no limiting predicate and changes every row", + remediation: "Add a WHERE condition that reads the target rows; a constant such as WHERE TRUE does not limit them. Use TRUNCATE, or an exception for this file, when changing every row is intended.", +}; +pub(crate) const INSERT_WITHOUT_COLUMNS: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN050", + message: "INSERT does not list its target columns", + remediation: "List the target columns, for example INSERT INTO orders (order_id, status) SELECT ..., so values cannot shift into the wrong column when the table changes.", +}; +pub(crate) const NATURAL_JOIN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN051", + message: "NATURAL JOIN matches on every column the relations share", + remediation: "Join with an explicit ON or USING condition; adding a column with a shared name to either relation silently changes a NATURAL JOIN.", +}; +pub(crate) const REVERSED_BETWEEN: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN052", + message: "BETWEEN lower bound is greater than its upper bound", + remediation: "BETWEEN low AND high is empty when low > high, so it never matches, and NOT BETWEEN matches every non-NULL value. Swap the bounds, or write BETWEEN SYMMETRIC where the dialect supports it.", +}; +pub(crate) const MISPLACED_AGGREGATE: LintRuleMetadata = LintRuleMetadata { + code: "SQLCLEAN053", + message: "Aggregate or window function is used in WHERE, ON or GROUP BY", + remediation: "Rows are filtered and grouped before aggregates and windows are computed. Filter an aggregate in HAVING, filter a window result in QUALIFY or an outer query, and group by the underlying columns.", +}; +pub(crate) const UNSTABLE_ROW_NUMBER_CODE: &str = "SQLCLEAN018"; +pub(crate) const UNPROVEN_RANKING_MESSAGE: &str = + "Ranking window order is not proven unique within its partition"; +pub(crate) const UNPROVEN_RANKING_REMEDIATION: &str = "PARTITION BY and ORDER BY must together cover a unique key of the ranked rows. End the ORDER BY with a unique, non-null tie-breaker column. If the rows have no unique column, build one upstream from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and order by it last."; +/// SQLCLEAN043 guidance before the `max_ranking_order_by` setting snippet. +pub(crate) const RANKING_SORT_GUIDANCE: &str = "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one upstream from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`), so SQLCLEAN018 can prove the order is deterministic. If the order genuinely needs more keys, raise `max_ranking_order_by`."; +/// SQLCLEAN044 guidance before the `max_literal_length` setting snippet. +pub(crate) const LONG_LITERAL_GUIDANCE: &str = "Shorten the literal, or move the value into a reference table or a query parameter so it is defined once."; +pub(crate) const RULES_THRESHOLDS_SECTION: &str = "rules.thresholds"; +pub(crate) const MAX_RANKING_ORDER_BY_KEY: &str = "max_ranking_order_by"; +pub(crate) const MAX_LITERAL_LENGTH_KEY: &str = "max_literal_length"; +pub(crate) const DISTINCT_UNUSED_REMEDIATION: &str = "This CTE uses SELECT DISTINCT, so the column still decides which rows are kept. Make the deduplication independent of the column, or restructure the CTE so it outputs only what later steps read."; +pub(crate) const GROUP_KEY_UNUSED_REMEDIATION: &str = "Drop the column from the CTE output but keep it in GROUP BY; the grouping grain stays the same."; +pub(crate) const GROUP_BY_ALL_UNUSED_REMEDIATION: &str = "This CTE uses GROUP BY ALL, so the column is part of the grouping grain. List the grouping columns explicitly and keep this one in GROUP BY, then drop it from the output."; +/// Expression kinds that compute one value per input row without changing the row set. +pub(crate) const ROW_NEUTRAL_EXPRESSIONS: &[&str] = &[ + "literal", + "boolean", + "null", + "identifier", + "column", + "paren", + "dot", + "subscript", + "data_type", + "interval", + "cast", + "try_cast", + "safe_cast", + "collation", + "case", + "and", + "or", + "not", + "neg", + "add", + "sub", + "mul", + "div", + "mod", + "eq", + "neq", + "lt", + "lte", + "gt", + "gte", + "like", + "i_like", + "is", + "is_null", + "is_true", + "is_false", + "in", + "between", + "null_safe_eq", + "bitwise_and", + "bitwise_or", + "bitwise_xor", + "concat", + "concat_ws", + "substring", + "upper", + "lower", + "length", + "trim", + "replace", + "left", + "right", + "lpad", + "rpad", + "split", + "split_part", + "regexp_replace", + "initcap", + "position", + "contains", + "starts_with", + "ends_with", + "abs", + "round", + "floor", + "ceil", + "greatest", + "least", + "coalesce", + "null_if", + "if_func", + "if_null", + "nvl", + "nvl2", + "decode", + "date_add", + "date_diff", + "date_trunc", + "timestamp_trunc", + "extract", + "to_date", + "to_timestamp", + "to_char", + "to_number", + "date", + "year", + "month", + "last_day", + "struct_extract", +]; +/// Call names SQLClean removed as deterministic scalar calls, at the arities the next two lists allow. +pub(crate) const DETERMINISTIC_SCALAR_FUNCTIONS: &[&str] = &[ + "ABS", + "ARRAY_SIZE", + "CEIL", + "CEILING", + "CHARACTER_LENGTH", + "CHARINDEX", + "CHAR_LENGTH", + "COALESCE", + "CONCAT", + "CONCAT_WS", + "CONTAINS", + "DATEADD", + "DATEDIFF", + "DATE_ADD", + "DATE_DIFF", + "DATE_PART", + "DATE_TRUNC", + "DIV0", + "ENDSWITH", + "ENDS_WITH", + "EXTRACT", + "FLOOR", + "GREATEST", + "IF", + "IFF", + "IFNULL", + "INITCAP", + "LAST_DAY", + "LEAST", + "LEFT", + "LEN", + "LENGTH", + "LOWER", + "LPAD", + "LTRIM", + "MD5", + "MONTH", + "NULLIF", + "NVL", + "NVL2", + "POWER", + "REGEXP_REPLACE", + "REPLACE", + "RIGHT", + "ROUND", + "RPAD", + "RTRIM", + "SPLIT_PART", + "STARTSWITH", + "STARTS_WITH", + "SUBSTR", + "SUBSTRING", + "TEXTLEN", + "TO_CHAR", + "TO_DATE", + "TO_NUMBER", + "TO_VARCHAR", + "TRIM", + "TRUNC", + "TRY_TO_NUMBER", + "UPPER", + "YEAR", + "ZEROIFNULL", +]; +/// Calls SQLClean removed only with one argument; polyglot left other arities as unknown calls. +pub(crate) const SINGLE_ARGUMENT_SCALAR_FUNCTIONS: &[&str] = &[ + "CHARACTER_LENGTH", + "CHAR_LENGTH", + "INITCAP", + "LEN", + "TEXTLEN", +]; +/// Calls SQLClean removed only with more than one argument; polyglot typed the one-argument form. +pub(crate) const MULTI_ARGUMENT_SCALAR_FUNCTIONS: &[&str] = &["ARRAY_SIZE"]; +pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: usize = 100; +pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: usize = 5; +pub(crate) const TIE_SENSITIVE_FUNCTIONS: [&str; 3] = ["ROW_NUMBER", "FIRST_VALUE", "LAST_VALUE"]; +pub(crate) const MIN_REMOVABLE_OUTPUTS: usize = 2; +pub(crate) const FIRST_ROW: &str = "1"; + +/// The keywords that end a select list. +pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ + TokenType::From, + TokenType::Where, + TokenType::Group, + TokenType::GroupBy, + TokenType::Having, + TokenType::Qualify, + TokenType::Order, + TokenType::OrderBy, + TokenType::Limit, + TokenType::Window, + TokenType::Union, + TokenType::Except, + TokenType::Intersect, +]; + +/// Other dialects' aggregates that Snowflake lacks, so lint cannot tell what a Snowflake call by the name is. +pub(crate) const NON_SNOWFLAKE_AGGREGATE_CALLS: [&str; 6] = [ + "BIT_AND", "BIT_OR", "BIT_XOR", "BOOL_AND", "BOOL_OR", "EVERY", +]; diff --git a/crates/sqlquack-lint/src/quality/main/diagnostics.rs b/crates/sqlquack-lint/src/quality/main/diagnostics.rs new file mode 100644 index 000000000..bc3ce2976 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/main/diagnostics.rs @@ -0,0 +1,33 @@ +use crate::lint::models::LintDiagnostic; +use crate::quality::constants::{ + LONG_LITERAL, RANKING_SORT_CAP, REPEATED_JSON_PARSE, UNSTABLE_ROW_NUMBER_CODE, + UNUSED_CTE_OUTPUT, +}; +use crate::quality::keys::{KeyEnvironment, key_environment}; +use crate::quality::models::QualityRequest; +use crate::quality::{cte_columns, json_parses, literals, ranking, safety}; + +/// Run every enabled SQL quality Rule over one parsed body. +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Result, String> { + if request.max_literal_length == 0 || request.max_ranking_order_by == 0 { + return Err("max_literal_length and max_ranking_order_by must be positive".to_owned()); + } + let mut found: Vec = Vec::new(); + if request.enabled.contains(LONG_LITERAL.code) { + found.extend(literals::diagnostics(request)); + } + if request.enabled.contains(REPEATED_JSON_PARSE.code) { + found.extend(json_parses::diagnostics(request)); + } + found.extend(safety::diagnostics(request)); + if request.enabled.contains(UNUSED_CTE_OUTPUT.code) { + found.extend(cte_columns::diagnostics(request)); + } + if request.enabled.contains(RANKING_SORT_CAP.code) + || request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE) + { + let environment: KeyEnvironment = key_environment(request); + found.extend(ranking::diagnostics(request, &environment)); + } + Ok(found) +} diff --git a/crates/sqlquack-lint/src/quality/main/mod.rs b/crates/sqlquack-lint/src/quality/main/mod.rs new file mode 100644 index 000000000..4c630487f --- /dev/null +++ b/crates/sqlquack-lint/src/quality/main/mod.rs @@ -0,0 +1 @@ +pub mod diagnostics; diff --git a/crates/sqlquack-lint/src/quality/mod.rs b/crates/sqlquack-lint/src/quality/mod.rs new file mode 100644 index 000000000..d01d8fda5 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/mod.rs @@ -0,0 +1,29 @@ +//! Enforceable SQL quality Rules with project-wide limits and ranking determinism proofs. + +pub mod constants; +#[path = "_helpers/cte_columns.rs"] +mod cte_columns; +#[path = "_helpers/json_parses.rs"] +mod json_parses; +#[path = "_helpers/keys.rs"] +mod keys; +#[path = "_helpers/literals.rs"] +mod literals; +pub mod main; +pub mod models; +#[path = "_helpers/projection_spans.rs"] +mod projection_spans; +#[path = "_helpers/ranking.rs"] +mod ranking; +#[path = "_helpers/removal_safety.rs"] +mod removal_safety; +#[path = "_helpers/safety.rs"] +mod safety; +#[path = "_helpers/setting_help.rs"] +mod setting_help; +#[path = "_helpers/syntax.rs"] +mod syntax; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-lint/src/quality/models.rs b/crates/sqlquack-lint/src/quality/models.rs new file mode 100644 index 000000000..18d4d6754 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/models.rs @@ -0,0 +1,22 @@ +use std::collections::{HashMap, HashSet}; + +use sqlquack_syntax::tokens::models::Token; +use sqlquack_syntax::{Dialect, Expression}; + +/// Inputs for one body's quality Rules, borrowed from the native lint request. +#[derive(Debug)] +pub(crate) struct QualityRequest<'a> { + pub(crate) statements: &'a [Expression], + pub(crate) tokens: &'a [Token], + pub(crate) dialect: Dialect, + pub(crate) enabled: &'a HashSet, + pub(crate) max_literal_length: usize, + pub(crate) max_ranking_order_by: usize, + pub(crate) relation_keys: &'a HashMap>>, + pub(crate) externally_referenced_ctes: &'a HashSet, + pub(crate) fixture_body: bool, + /// The configuration file remediation text tells the user to edit. + pub(crate) config_file: &'a str, + /// The advice a long literal finding gives before its limit setting. + pub(crate) long_literal_guidance: &'a str, +} diff --git a/crates/sqlquack-lint/src/quality/tests.rs b/crates/sqlquack-lint/src/quality/tests.rs new file mode 100644 index 000000000..8b028341c --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_json_parses.rs"] +mod json_parses; +#[path = "tests/test_quality_rules.rs"] +mod quality_rules; +#[path = "tests/test_quality_scale.rs"] +mod quality_scale; +#[path = "tests/test_safety_rules.rs"] +mod safety_rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-lint/src/quality/tests/helpers.rs b/crates/sqlquack-lint/src/quality/tests/helpers.rs new file mode 100644 index 000000000..52c804571 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/helpers.rs @@ -0,0 +1,143 @@ +use serde_json::{Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::quality::tests::test_types::QualityRuleTestCase; + +pub(crate) fn lint(test_case: &QualityRuleTestCase) -> Result, String> { + lint_with_host_fields(test_case, "{}") +} + +/// Lint `test_case` with `host_fields`, a JSON object, merged into the request. +pub(crate) fn lint_with_host_fields( + test_case: &QualityRuleTestCase, + host_fields: &str, +) -> Result, String> { + let relation_keys: Value = + serde_json::from_str(test_case.relation_keys).map_err(|error| error.to_string())?; + let host_fields: Value = + serde_json::from_str(host_fields).map_err(|error| error.to_string())?; + let mut request: Value = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": [test_case.rule], + "max_literal_length": 24, + "max_ranking_order_by": 2, + "relation_keys": relation_keys, + }); + request + .as_object_mut() + .ok_or("the request is not an object")? + .extend( + host_fields + .as_object() + .cloned() + .ok_or("host fields are not an object")?, + ); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default()) +} + +/// Lint `sql` in `dialect` with only `rule` enabled. +pub(crate) fn lint_dialect(sql: &str, dialect: &str, rule: &str) -> Result, String> { + let request: Value = json!({ + "version": 1, + "sql": sql, + "dialect": dialect, + "enabled_rules": [rule], + }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default()) +} + +/// One DuckDB SELECT that parses each of `column_count` payload columns twice. +pub(crate) fn wide_json_select(column_count: usize) -> String { + let projections: Vec = (0..column_count * 2) + .map(|index| format!("json(e.c{})->>'f{index}' AS f{index}", index % column_count)) + .collect(); + format!("SELECT {} FROM events AS e", projections.join(", ")) +} + +pub(crate) fn anchors(sql: &str, diagnostics: &[Value]) -> Vec { + diagnostics + .iter() + .map(|diagnostic| { + let start: usize = diagnostic["start"].as_u64().unwrap_or(0) as usize; + let end: usize = diagnostic["end"].as_u64().unwrap_or(0) as usize; + sql.chars() + .skip(start) + .take(end.saturating_sub(start)) + .collect() + }) + .collect() +} + +pub(crate) fn fixed(sql: &str, diagnostics: &[Value]) -> Option { + let mut edits: Vec<(usize, usize, String)> = diagnostics + .iter() + .filter_map(|diagnostic| { + let fix: &Value = diagnostic.get("fix")?; + Some(( + fix["start"].as_u64()? as usize, + fix["end"].as_u64()? as usize, + fix["replacement"].as_str()?.to_owned(), + )) + }) + .collect(); + edits.sort_by_key(|edit| std::cmp::Reverse(edit.0)); + let applied: bool = !edits.is_empty(); + let mut characters: Vec = sql.chars().collect(); + for (start, end, replacement) in edits { + characters.splice(start..end, replacement.chars()); + } + applied.then(|| characters.into_iter().collect()) +} + +/// A chain of CTEs that each drop their last column and rank without a declared key. +pub(crate) fn wide_cte_chain(cte_count: usize, column_count: usize) -> String { + let columns: Vec = (0..column_count).map(|index| format!("c{index}")).collect(); + let mut ctes: Vec = vec![format!( + "step_0 AS (SELECT {} FROM source_rows AS s)", + columns + .iter() + .map(|column| format!("s.{column}")) + .collect::>() + .join(", ") + )]; + for step in 1..cte_count { + let mut outputs: Vec = columns[..column_count - 1] + .iter() + .map(|column| format!("p.{column}")) + .collect(); + outputs.push(format!( + "ROW_NUMBER() OVER (PARTITION BY p.c0 ORDER BY p.c1) AS c{}", + column_count - 1 + )); + ctes.push(format!( + "step_{step} AS (SELECT {} FROM step_{} AS p)", + outputs.join(", "), + step - 1 + )); + } + format!( + "WITH {} SELECT * FROM step_{}", + ctes.join(",\n"), + cte_count - 1 + ) +} + +/// Whether lint reports `dialect` as unavailable, because SQLQuack does not parse it yet. +pub(crate) fn lint_unavailable(dialect: &str) -> Result { + let request: Value = json!({ "version": 1, "sql": "SELECT 1", "dialect": dialect }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response.get("unavailable").is_some()) +} diff --git a/crates/sqlquack-lint/src/quality/tests/test_json_parses.rs b/crates/sqlquack-lint/src/quality/tests/test_json_parses.rs new file mode 100644 index 000000000..a5cef7b69 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/test_json_parses.rs @@ -0,0 +1,292 @@ +use std::time::Instant; + +use serde_json::Value; + +use crate::quality::tests::helpers::{anchors, lint_dialect, lint_unavailable, wide_json_select}; +use crate::quality::tests::test_types::{JsonParseScaleTestCase, JsonParseTestCase}; + +const RULE: &str = "SQLCLEAN045"; + +#[test] +fn given_json_parse_calls_when_linting_then_repeats_in_one_select_are_reported() +-> Result<(), String> { + let test_cases = [ + JsonParseTestCase { + description: "same column parsed twice reports the second call", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.value):order_id AS order_id, TRY_PARSE_JSON(e.value):status AS status FROM events AS e", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "the finding is located at the second call", + dialect: "snowflake", + sql: "SELECT try_parse_json(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "three parses give one finding with the count", + dialect: "snowflake", + sql: "SELECT PARSE_JSON(value):a AS a, PARSE_JSON(value):b AS b FROM events WHERE PARSE_JSON(value):c IS NOT NULL", + expected_anchors: &["PARSE_JSON"], + expected_details: &["PARSE_JSON(value) is parsed 3 times in one SELECT."], + }, + JsonParseTestCase { + description: "column casing and wrappers do not hide a repeat", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(LOWER(E.Value)):a AS a, TRY_PARSE_JSON((CAST(TRIM(e.value) AS VARCHAR))):b AS b, TRY_PARSE_JSON(COALESCE(UPPER(e.VALUE))):c AS c FROM events AS e", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(e.value) is parsed 3 times in one SELECT."], + }, + JsonParseTestCase { + description: "multi-argument COALESCE is not a plain column", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(COALESCE(e.value, e.backup)):a AS a, TRY_PARSE_JSON(COALESCE(e.value, e.fallback)):b AS b, TRY_PARSE_JSON(e.value):c AS c FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "TRIM with characters is not a plain column", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(TRIM(e.value, ' ')):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "quoted names differing in case are different columns", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.\"Value\"):a AS a, TRY_PARSE_JSON(e.\"value\"):b AS b FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "quoted and unquoted names are not merged", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.\"value\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "quoted qualifiers differing in case are different relations", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(\"E\".value):a AS a, TRY_PARSE_JSON(\"e\".value):b AS b FROM events AS \"E\" INNER JOIN events AS \"e\" ON \"E\".id = \"e\".id", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "identical quoted names are reported as written", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(\"E\".\"Value\"):a AS a, TRY_PARSE_JSON(\"E\".\"Value\"):b AS b FROM events AS \"E\"", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(\"E\".\"Value\") is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "snowflake quoted upper case matches the folded unquoted name", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.\"VALUE\"):a AS a, TRY_PARSE_JSON(e.value):b AS b FROM events AS e", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(e.value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "duckdb quoted names are case-insensitive", + dialect: "duckdb", + sql: "SELECT json(e.\"Value\") -> 'a' AS a, json(e.value) -> 'b' AS b FROM events AS e", + expected_anchors: &["json"], + expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "references are grouped as written", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(e.value):b AS b, TRY_PARSE_JSON(events.value):c AS c FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "different columns pass", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.payload):a AS a, TRY_PARSE_JSON(e.metadata):b AS b FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "same column name on different tables passes", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(o.payload):a AS a, TRY_PARSE_JSON(c.payload):b AS b FROM orders AS o INNER JOIN customers AS c ON o.customer_id = c.customer_id", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "different parse functions are not merged", + dialect: "snowflake", + sql: "SELECT PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "one parse per CTE scope passes", + dialect: "snowflake", + sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events), final AS (SELECT TRY_PARSE_JSON(value) AS payload FROM events) SELECT payload FROM final", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "nested SELECT is its own scope", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(e.value):a AS a, (SELECT MAX(TRY_PARSE_JSON(x.value):b) FROM events AS x) AS b FROM events AS e", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "repeat inside a subquery is reported in that scope", + dialect: "snowflake", + sql: "SELECT e.order_id FROM events AS e WHERE e.order_id IN (SELECT TRY_PARSE_JSON(value):id FROM events WHERE TRY_PARSE_JSON(value):ok)", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "union branches are separate scopes", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(value) AS payload FROM events UNION ALL SELECT TRY_PARSE_JSON(value) AS payload FROM events", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "a repeat inside one CTE body is reported", + dialect: "snowflake", + sql: "WITH parsed AS (SELECT TRY_PARSE_JSON(value):a AS a, TRY_PARSE_JSON(value):b AS b FROM events) SELECT a, b FROM parsed", + expected_anchors: &["TRY_PARSE_JSON"], + expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "non-column arguments pass", + dialect: "snowflake", + sql: "SELECT TRY_PARSE_JSON(value || '') AS a, TRY_PARSE_JSON(value || '') AS b FROM events", + expected_anchors: &[], + expected_details: &[], + }, + JsonParseTestCase { + description: "duckdb json is matched", + dialect: "duckdb", + sql: "SELECT json(e.value)->>'a' AS a, json(e.value)->>'b' AS b FROM events AS e", + expected_anchors: &["json"], + expected_details: &["JSON(e.value) is parsed 2 times in one SELECT."], + }, + ]; + let unparsed_dialect_cases = [ + JsonParseTestCase { + description: "bigquery quoted names keep backticks", + dialect: "bigquery", + sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.`Value`), '$.b') AS b FROM events AS e", + expected_anchors: &["PARSE_JSON"], + expected_details: &["PARSE_JSON(e.`Value`) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "bigquery backticked names are case-insensitive", + dialect: "bigquery", + sql: "SELECT JSON_VALUE(PARSE_JSON(e.`Value`), '$.a') AS a, JSON_VALUE(PARSE_JSON(e.value), '$.b') AS b FROM events AS e", + expected_anchors: &["PARSE_JSON"], + expected_details: &["PARSE_JSON(e.value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "databricks backticked names are case-insensitive", + dialect: "databricks", + sql: "SELECT try_parse_json(e.`v`) AS a, try_parse_json(e.V) AS b FROM events AS e", + expected_anchors: &["try_parse_json"], + expected_details: &["TRY_PARSE_JSON(e.V) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "bigquery safe parse is matched", + dialect: "bigquery", + sql: "SELECT JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.a') AS a, JSON_VALUE(SAFE.PARSE_JSON(e.value), '$.b') AS b FROM events AS e", + expected_anchors: &["PARSE_JSON"], + expected_details: &["SAFE.PARSE_JSON(e.value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "databricks try parse is matched", + dialect: "databricks", + sql: "SELECT try_parse_json(value) AS a, try_parse_json(value) AS b FROM events", + expected_anchors: &["try_parse_json"], + expected_details: &["TRY_PARSE_JSON(value) is parsed 2 times in one SELECT."], + }, + JsonParseTestCase { + description: "dialects without an equivalent are not checked", + dialect: "postgres", + sql: "SELECT CAST(value AS JSONB) AS a, CAST(value AS JSONB) AS b FROM events", + expected_anchors: &[], + expected_details: &[], + }, + ]; + for test_case in &unparsed_dialect_cases { + assert!( + lint_unavailable(test_case.dialect)?, + "{}: lint is unavailable until SQLQuack parses {}", + test_case.description, + test_case.dialect + ); + } + for test_case in &test_cases { + let diagnostics: Vec = lint_dialect(test_case.sql, test_case.dialect, RULE)?; + assert_eq!( + anchors(test_case.sql, &diagnostics), + test_case.expected_anchors, + "{}", + test_case.description + ); + let details: Vec = diagnostics + .iter() + .map(|diagnostic| { + diagnostic["remediation"] + .as_str() + .unwrap_or_default() + .split_inclusive(". ") + .next() + .unwrap_or_default() + .trim_end() + .to_owned() + }) + .collect(); + assert_eq!( + details, test_case.expected_details, + "{}", + test_case.description + ); + assert!( + diagnostics + .iter() + .all(|diagnostic| diagnostic["fix"].is_null()), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_wide_select_with_many_repeated_parses_when_linting_then_it_stays_linear() +-> Result<(), String> { + let test_cases = [JsonParseScaleTestCase { + description: "3,000 projections parsing 1,500 payload columns twice each", + column_count: 1_500, + expected_findings: 1_500, + expected_max_seconds: 10.0, + }]; + for test_case in &test_cases { + let started: Instant = Instant::now(); + let diagnostics: Vec = + lint_dialect(&wide_json_select(test_case.column_count), "duckdb", RULE)?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + assert_eq!( + diagnostics.len(), + test_case.expected_findings, + "{}", + test_case.description + ); + assert!( + elapsed <= test_case.expected_max_seconds, + "{}: {elapsed:.2}s", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/quality/tests/test_quality_rules.rs b/crates/sqlquack-lint/src/quality/tests/test_quality_rules.rs new file mode 100644 index 000000000..aaf6ea1b4 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/test_quality_rules.rs @@ -0,0 +1,472 @@ +use serde_json::Value; + +use crate::quality::tests::helpers::{anchors, fixed, lint, lint_with_host_fields}; +use crate::quality::tests::test_types::{QualityRemediationTestCase, QualityRuleTestCase}; + +#[test] +fn given_quality_rule_inputs_when_linting_then_findings_and_fixes_match() -> Result<(), String> { + let test_cases = [ + QualityRuleTestCase { + description: "long literal is reported without a split fix", + sql: "SELECT 'pending orders awaiting shipment' AS label", + rule: "SQLCLEAN044", + relation_keys: "{}", + expected_anchors: &["'pending orders awaiting shipment'"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "short literal passes", + sql: "SELECT 'pending' AS label", + rule: "SQLCLEAN044", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "typed literal is reported without a fix", + sql: "SELECT TIMESTAMP '2026-04-01 10:00:00.000000000' AS ordered_at", + rule: "SQLCLEAN044", + relation_keys: "{}", + expected_anchors: &["'2026-04-01 10:00:00.000000000'"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ranking sort above the cap", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", + rule: "SQLCLEAN043", + relation_keys: "{}", + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ranking sort within the cap", + sql: "SELECT RANK() OVER (ORDER BY o.a, o.order_id) AS rn FROM orders AS o", + rule: "SQLCLEAN043", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ordering covers a declared key", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at, o.order_id) AS rn FROM orders AS o", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ordering without a key is unproven", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) AS rn FROM orders AS o", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "key proven through a CTE and a one-to-one join", + sql: "WITH lines AS (SELECT l.line_id, l.order_id FROM order_lines AS l), final AS (SELECT l.line_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY l.line_id) AS rn FROM lines AS l INNER JOIN orders AS o ON l.order_id = o.order_id) SELECT f.line_id, f.customer_id, f.rn FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "key lost through a fan-out join", + sql: "WITH final AS (SELECT o.order_id, ROW_NUMBER() OVER (ORDER BY o.order_id) AS rn FROM orders AS o INNER JOIN order_lines AS l ON l.order_id = o.order_id) SELECT f.order_id, f.rn FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "GROUP BY grain is a key", + sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "positional GROUP BY grain is a key", + sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY 1), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a later rn = 1 filter makes the partition a key", + sql: "WITH ranked AS (SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a QUALIFY window cannot prove itself", + sql: "SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount DESC) = 1", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a QUALIFY partition is a key for later steps", + sql: "WITH latest AS (SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", + rule: "SQLCLEAN018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "RANK ties are deterministic", + sql: "SELECT RANK() OVER (ORDER BY o.ordered_at) AS rn FROM orders AS o", + rule: "SQLCLEAN018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "unused CTE output is removed", + sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "star pass-through into the final CTE keeps every column", + sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT * FROM orders) SELECT * FROM final", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "star pass-through follows the reader's own reads", + sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "a same-named cast of a source column is not a lateral read", + sql: "WITH orders AS (SELECT o.order_id, CAST(note AS VARCHAR) AS note, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["CAST(note AS VARCHAR) AS note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "an interpolated expression in a reader may read any column", + sql: "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r WHERE __sqlclean_relation_parameter_0__) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an interpolated output is never reported", + sql: "WITH orders AS (SELECT o.order_id, __SQLCLEAN_INTERPOLATION_0002 FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT CTE reports without a fix", + sql: "WITH orders AS (SELECT DISTINCT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "columns read in later join and filter clauses are used", + sql: "WITH orders AS (SELECT o.order_id, o.customer_id, o.status FROM raw_orders AS o), final AS (SELECT c.name FROM orders AS r INNER JOIN customers AS c ON r.customer_id = c.customer_id WHERE r.status = 'open' QUALIFY ROW_NUMBER() OVER (PARTITION BY r.order_id ORDER BY c.name) = 1) SELECT f.name FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a column read only through a nested star is used", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g.*) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a whole-row reference reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COUNT(DISTINCT row) reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT COUNT(DISTINCT g) AS n FROM g) SELECT f.n FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COLUMNS(*) reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id FROM g WHERE COLUMNS(*) IS NOT NULL) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COLUMNS with a pattern reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT MAX(COLUMNS('order.*')) FROM g) SELECT * FROM final", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT star pass-through reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT DISTINCT * FROM g), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a positional ORDER BY over a star pass-through reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT * FROM g ORDER BY 2 LIMIT 1), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an output alias read in WHERE or GROUP BY is used", + sql: "WITH g AS (SELECT r.order_id, r.amount * 2 AS doubled, r.customer_id % 2 AS bucket FROM raw_orders AS r WHERE doubled > 1 GROUP BY r.order_id, doubled, bucket), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "GROUP BY ALL grouping column is reported without a fix", + sql: "WITH g AS (SELECT r.customer_id, r.status, SUM(r.amount) AS total FROM raw_orders AS r GROUP BY ALL), final AS (SELECT g.customer_id, g.total FROM g) SELECT f.customer_id, f.total FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["r.status"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an aggregate in an ungrouped select is reported without a fix", + sql: "WITH g AS (SELECT 'all' AS label, COUNT(*) AS n FROM raw_orders AS r), final AS (SELECT g.label FROM g) SELECT f.label FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["COUNT(*) AS n"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a set-returning projection is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, UNNEST(r.tags) AS tag FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["UNNEST(r.tags) AS tag"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ORDER BY ALL with LIMIT is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, r.customer_id FROM raw_orders AS r ORDER BY ALL LIMIT 1), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["r.customer_id"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT ON is reported without a fix", + sql: "WITH g AS (SELECT DISTINCT ON (r.customer_id) r.order_id, r.amount FROM raw_orders AS r ORDER BY r.customer_id, r.order_id), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["r.amount"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a volatile projection is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, RANDOM() AS draw FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["RANDOM() AS draw"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a deterministic scalar projection is removed", + sql: "WITH g AS (SELECT r.order_id, COALESCE(UPPER(r.status), 'none') AS label FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["COALESCE(UPPER(r.status), 'none') AS label"], + expected_fixed_sql: Some( + "WITH g AS (SELECT r.order_id FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + ), + }, + ]; + for test_case in &test_cases { + let diagnostics: Vec = lint(test_case)?; + assert_eq!( + anchors(test_case.sql, &diagnostics), + test_case.expected_anchors, + "{}", + test_case.description + ); + assert_eq!( + fixed(test_case.sql, &diagnostics).as_deref(), + test_case.expected_fixed_sql, + "{}", + test_case.description + ); + } + // Snowflake keeps these calls by their written name, so they reach the call-name list. + let snowflake_cases = [ + QualityRuleTestCase { + description: "SQRT is reported without a fix, as SQLClean read it", + sql: "WITH g AS (SELECT r.order_id, SQRT(r.amount) AS root FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["SQRT(r.amount) AS root"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "INITCAP is removed, as SQLClean read it", + sql: "WITH g AS (SELECT r.order_id, INITCAP(r.status) AS label FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQLCLEAN042", + relation_keys: "{}", + expected_anchors: &["INITCAP(r.status) AS label"], + expected_fixed_sql: Some( + "WITH g AS (SELECT r.order_id FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + ), + }, + ]; + for test_case in &snowflake_cases { + let diagnostics: Vec = + lint_with_host_fields(test_case, r#"{"dialect": "snowflake"}"#)?; + assert_eq!( + anchors(test_case.sql, &diagnostics), + test_case.expected_anchors, + "{}", + test_case.description + ); + assert_eq!( + fixed(test_case.sql, &diagnostics).as_deref(), + test_case.expected_fixed_sql, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_threshold_findings_when_linting_then_remediation_shows_the_exact_setting() +-> Result<(), String> { + let test_cases = [ + QualityRemediationTestCase { + description: "long literal points at shortening it and the length setting", + sql: "SELECT 'pending orders awaiting shipment' AS label", + rule: "SQLCLEAN044", + host_fields: "{}", + expected_fragments: &[ + "Shorten the literal", + "set this in sqlclean.toml:", + "the current value is 24", + "[rules.thresholds]\n max_literal_length = 32", + ], + absent_fragments: &[], + }, + QualityRemediationTestCase { + description: "a host's long literal guidance replaces SQLClean's and names its file", + sql: "SELECT 'pending orders awaiting shipment' AS label", + rule: "SQLCLEAN044", + host_fields: r#"{"long_literal_guidance": "Declare it once as a constant.", "config_file": "host.toml"}"#, + expected_fragments: &[ + "Declare it once as a constant. To allow literals of 32 characters", + "set this in host.toml:", + ], + absent_fragments: &["Shorten the literal", "sqlclean.toml"], + }, + QualityRemediationTestCase { + description: "long ranking sort points at a row key and the sort setting", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", + rule: "SQLCLEAN043", + host_fields: "{}", + expected_fragments: &[ + "unique row key as the final tie-breaker", + "HASH(source_partition, source_offset, line_index) AS row_key", + "the current value is 2", + "[rules.thresholds]\n max_ranking_order_by = 3", + ], + absent_fragments: &[], + }, + QualityRemediationTestCase { + description: "unproven ranking points at a unique tie-breaker and a built row key", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a) AS rn FROM orders AS o", + rule: "SQLCLEAN018", + host_fields: "{}", + expected_fragments: &[ + "cover a unique key of the ranked rows", + "a unique, non-null tie-breaker column", + "HASH(source_partition, source_offset, line_index) AS row_key", + ], + absent_fragments: &[], + }, + ]; + for test_case in &test_cases { + let diagnostics: Vec = lint_with_host_fields( + &QualityRuleTestCase { + description: test_case.description, + sql: test_case.sql, + rule: test_case.rule, + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + test_case.host_fields, + )?; + let remediation: &str = diagnostics + .first() + .and_then(|diagnostic| diagnostic["remediation"].as_str()) + .ok_or_else(|| format!("{}: no finding", test_case.description))?; + let present: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| remediation.contains(fragment)) + .collect(); + assert_eq!( + present, test_case.expected_fragments, + "{}: {remediation}", + test_case.description + ); + let leaked: Vec<&str> = test_case + .absent_fragments + .iter() + .copied() + .filter(|fragment| remediation.contains(fragment)) + .collect(); + assert_eq!( + leaked, + Vec::<&str>::new(), + "{}: {remediation}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/quality/tests/test_quality_scale.rs b/crates/sqlquack-lint/src/quality/tests/test_quality_scale.rs new file mode 100644 index 000000000..4505bc561 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/test_quality_scale.rs @@ -0,0 +1,43 @@ +use std::time::Instant; + +use serde_json::{Value, json}; + +use crate::lint::main::engine::lint_json; +use crate::quality::tests::helpers::wide_cte_chain; +use crate::quality::tests::test_types::QualityScaleTestCase; + +#[test] +fn given_many_wide_ctes_with_findings_when_linting_then_quality_rules_stay_linear() +-> Result<(), String> { + let test_cases = [QualityScaleTestCase { + description: "1,500 wide CTEs, each dropping a column and ranking without a key", + cte_count: 1_500, + column_count: 24, + expected_findings: 2_998, + expected_max_seconds: 20.0, + }]; + for test_case in &test_cases { + let request: Value = json!({ + "version": 1, + "sql": wide_cte_chain(test_case.cte_count, test_case.column_count), + "dialect": "duckdb", + "enabled_rules": ["SQLCLEAN018", "SQLCLEAN042", "SQLCLEAN043", "SQLCLEAN044"], + }); + let started: Instant = Instant::now(); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + assert_eq!( + response["diagnostics"].as_array().map_or(0, Vec::len), + test_case.expected_findings, + "{}", + test_case.description + ); + assert!( + elapsed <= test_case.expected_max_seconds, + "{}: {elapsed:.2}s", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/quality/tests/test_safety_rules.rs b/crates/sqlquack-lint/src/quality/tests/test_safety_rules.rs new file mode 100644 index 000000000..96321d7e5 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/test_safety_rules.rs @@ -0,0 +1,217 @@ +use serde_json::Value; + +use crate::quality::tests::helpers::{anchors, lint_dialect, lint_unavailable}; +use crate::quality::tests::test_types::SafetyRuleTestCase; + +#[test] +fn given_safety_rule_inputs_when_linting_then_exactly_the_mistakes_are_reported() +-> Result<(), String> { + let test_cases = [ + SafetyRuleTestCase { + description: "unquoted CTE names collide regardless of case", + dialect: "duckdb", + sql: "WITH paid AS (SELECT 1 AS x), PAID AS (SELECT 2 AS x) SELECT x FROM paid", + rule: "SQLCLEAN046", + expected_anchors: &["PAID"], + }, + SafetyRuleTestCase { + description: "a quoted name equals the folded unquoted name in Snowflake", + dialect: "snowflake", + sql: r#"WITH "PAID" AS (SELECT 1 AS x), paid AS (SELECT 2 AS x) SELECT x FROM paid"#, + rule: "SQLCLEAN046", + expected_anchors: &["paid"], + }, + SafetyRuleTestCase { + description: "a nested WITH may reuse an outer CTE name", + dialect: "duckdb", + sql: "WITH paid AS (WITH paid AS (SELECT 1 AS x) SELECT x FROM paid) SELECT x FROM paid", + rule: "SQLCLEAN046", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "simple CASE compares with NULL, and a singleton IN list holds only NULL", + dialect: "duckdb", + sql: "SELECT CASE status WHEN 'open' THEN 1 WHEN NULL THEN 2 END AS s FROM orders WHERE region IN (NULL)", + rule: "SQLCLEAN047", + expected_anchors: &["WHEN NULL", "IN (NULL)"], + }, + SafetyRuleTestCase { + description: "searched CASE, IS NULL, NOT IN and a longer IN list pass", + dialect: "duckdb", + sql: "SELECT CASE WHEN status IS NULL THEN 1 END AS s FROM orders WHERE region NOT IN (NULL) AND kind IN (1, NULL)", + rule: "SQLCLEAN047", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "only the CASE whose operand compares with NULL is reported", + dialect: "duckdb", + sql: "SELECT CASE status WHEN 'open' THEN CASE WHEN NULL THEN 1 END END AS a, CASE WHEN amount > 0 THEN CASE region WHEN NULL THEN 2 END END AS b FROM orders", + rule: "SQLCLEAN047", + expected_anchors: &["WHEN NULL"], + }, + SafetyRuleTestCase { + description: "division and modulo by literal zero in operator and function form", + dialect: "duckdb", + sql: "SELECT amount / 0 AS a, amount % 0.0 AS b, MOD(amount, 0) AS c, amount / (-0) AS d FROM orders", + rule: "SQLCLEAN048", + expected_anchors: &["/ 0", "% 0.0", "MOD(amount, 0)", "/ (-0)"], + }, + SafetyRuleTestCase { + description: "a non-zero or computed divisor passes", + dialect: "duckdb", + sql: "SELECT amount / 0.5 AS a, amount / 1e-400 AS e, amount / 0 ** 2 AS b, MOD(amount, 0 + 1) AS c, amount / NULLIF(quantity, 0) AS d FROM orders", + rule: "SQLCLEAN048", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "a missing or constant WHERE does not limit an UPDATE or DELETE", + dialect: "duckdb", + sql: "UPDATE orders SET status = 'closed'; DELETE FROM orders WHERE TRUE; DELETE FROM orders WHERE 1 = 1", + rule: "SQLCLEAN049", + expected_anchors: &["UPDATE", "DELETE", "DELETE"], + }, + SafetyRuleTestCase { + description: "a row predicate or a subquery limits the write", + dialect: "duckdb", + sql: "UPDATE orders SET status = 'closed' WHERE closed_at IS NOT NULL; DELETE FROM orders WHERE EXISTS (SELECT 1 FROM refunds)", + rule: "SQLCLEAN049", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "an INSERT without a column list in query and VALUES form", + dialect: "duckdb", + sql: "INSERT INTO orders SELECT * FROM staged_orders; INSERT INTO orders VALUES (1, 'open')", + rule: "SQLCLEAN050", + expected_anchors: &["INSERT", "INSERT"], + }, + SafetyRuleTestCase { + description: "listed columns and BY NAME pass", + dialect: "duckdb", + sql: "INSERT INTO orders (order_id) SELECT order_id FROM staged_orders; INSERT INTO orders BY NAME SELECT 1 AS order_id", + rule: "SQLCLEAN050", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "NATURAL joins are reported with their join keywords", + dialect: "duckdb", + sql: "SELECT * FROM orders NATURAL JOIN customers NATURAL LEFT OUTER JOIN regions JOIN items USING (order_id)", + rule: "SQLCLEAN051", + expected_anchors: &["NATURAL JOIN", "NATURAL LEFT OUTER JOIN"], + }, + SafetyRuleTestCase { + description: "reversed number, negative, decimal and date bounds, with or without NOT", + dialect: "duckdb", + sql: "SELECT * FROM orders WHERE a BETWEEN 10 AND 1 OR b NOT BETWEEN -1 AND -5 OR c BETWEEN 1.5 AND 1.25 OR d BETWEEN DATE '2026-02-01' AND DATE '2026-01-31'", + rule: "SQLCLEAN052", + expected_anchors: &[ + "BETWEEN 10 AND 1", + "BETWEEN -1 AND -5", + "BETWEEN 1.5 AND 1.25", + "BETWEEN DATE '2026-02-01' AND DATE '2026-01-31'", + ], + }, + SafetyRuleTestCase { + description: "ordered, equal, SYMMETRIC, computed, text and frame bounds pass", + dialect: "duckdb", + sql: "SELECT SUM(a) OVER (ORDER BY b ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) AS s FROM orders WHERE a BETWEEN 9 AND 10 OR b BETWEEN -5 AND -1 OR c BETWEEN 5 AND 5 OR d BETWEEN SYMMETRIC 10 AND 1 OR e BETWEEN 10 AND 1 + 20 OR f BETWEEN 'b' AND 'a'", + rule: "SQLCLEAN052", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "aggregates and windows in WHERE, ON and after a GROUP BY comma", + dialect: "duckdb", + sql: "SELECT o.customer_id FROM orders AS o JOIN customers AS c ON c.id = o.customer_id AND COUNT(*) > 1 WHERE SUM(o.amount) > 100 AND ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) = 1 GROUP BY o.customer_id, MAX(o.amount)", + rule: "SQLCLEAN053", + expected_anchors: &[ + "COUNT(*)", + "SUM(o.amount)", + "ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at)", + "MAX(o.amount)", + ], + }, + SafetyRuleTestCase { + description: "HAVING, QUALIFY, FILTER, subqueries and unknown functions pass", + dialect: "duckdb", + sql: "SELECT customer_id, COUNT(*) FILTER (WHERE amount > 0) AS paid FROM orders WHERE amount > (SELECT AVG(amount) FROM orders) AND order_total(amount) > 0 GROUP BY customer_id HAVING COUNT(*) > 1 QUALIFY ROW_NUMBER() OVER (ORDER BY customer_id) = 1", + rule: "SQLCLEAN053", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "DuckDB PIVOT USING aggregates end the pivot ON clause", + dialect: "duckdb", + sql: "PIVOT orders ON status IN ('placed') USING MAX(amount), SUM(amount) GROUP BY customer_id", + rule: "SQLCLEAN053", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "a join ON aggregate is still reported beside a pivot", + dialect: "duckdb", + sql: "SELECT o.id FROM orders AS o JOIN (PIVOT items ON kind USING MAX(qty)) AS p ON p.id = o.id AND MIN(o.amount) > 0", + rule: "SQLCLEAN053", + expected_anchors: &["MIN(o.amount)"], + }, + ]; + let unparsed_dialect_cases = [ + SafetyRuleTestCase { + description: "quoted names that differ in case are distinct in PostgreSQL", + dialect: "postgres", + sql: r#"WITH "paid" AS (SELECT 1 AS x), "PAID" AS (SELECT 2 AS x) SELECT x FROM "paid""#, + rule: "SQLCLEAN046", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "a joined DELETE and a LIMIT bound the rows in MySQL", + dialect: "mysql", + sql: "DELETE o FROM orders o JOIN customers c ON c.id = o.customer_id; DELETE FROM orders LIMIT 10", + rule: "SQLCLEAN049", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "UPDATE FROM without WHERE, and DELETE after a WITH clause", + dialect: "postgres", + sql: "UPDATE orders SET status = 'closed' FROM customers; WITH stale AS (SELECT 1 AS id) DELETE FROM orders", + rule: "SQLCLEAN049", + expected_anchors: &["UPDATE", "DELETE"], + }, + SafetyRuleTestCase { + description: "DEFAULT VALUES needs no column list", + dialect: "postgres", + sql: "INSERT INTO orders DEFAULT VALUES", + rule: "SQLCLEAN050", + expected_anchors: &[], + }, + SafetyRuleTestCase { + description: "ASYMMETRIC keeps the ordered meaning in PostgreSQL", + dialect: "postgres", + sql: "SELECT * FROM orders WHERE a BETWEEN ASYMMETRIC 10 AND 1", + rule: "SQLCLEAN052", + expected_anchors: &["BETWEEN ASYMMETRIC 10 AND 1"], + }, + SafetyRuleTestCase { + description: "multi-argument MAX is the scalar function in SQLite", + dialect: "sqlite", + sql: "SELECT order_id FROM orders WHERE max(amount, refunded) > 100", + rule: "SQLCLEAN053", + expected_anchors: &[], + }, + ]; + for test_case in &unparsed_dialect_cases { + assert!( + lint_unavailable(test_case.dialect)?, + "{}: lint is unavailable until SQLQuack parses {}", + test_case.description, + test_case.dialect + ); + } + for test_case in &test_cases { + let diagnostics: Vec = + lint_dialect(test_case.sql, test_case.dialect, test_case.rule)?; + assert_eq!( + anchors(test_case.sql, &diagnostics), + test_case.expected_anchors, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlquack-lint/src/quality/tests/test_types.rs b/crates/sqlquack-lint/src/quality/tests/test_types.rs new file mode 100644 index 000000000..6c02eab67 --- /dev/null +++ b/crates/sqlquack-lint/src/quality/tests/test_types.rs @@ -0,0 +1,49 @@ +pub(crate) struct QualityRuleTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) rule: &'static str, + pub(crate) relation_keys: &'static str, + pub(crate) expected_anchors: &'static [&'static str], + pub(crate) expected_fixed_sql: Option<&'static str>, +} + +pub(crate) struct QualityScaleTestCase { + pub(crate) description: &'static str, + pub(crate) cte_count: usize, + pub(crate) column_count: usize, + pub(crate) expected_findings: usize, + pub(crate) expected_max_seconds: f64, +} + +pub(crate) struct QualityRemediationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) rule: &'static str, + /// Extra request fields as a JSON object, as a host sends them. + pub(crate) host_fields: &'static str, + pub(crate) expected_fragments: &'static [&'static str], + pub(crate) absent_fragments: &'static [&'static str], +} + +pub(crate) struct JsonParseTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_anchors: &'static [&'static str], + pub(crate) expected_details: &'static [&'static str], +} + +pub(crate) struct JsonParseScaleTestCase { + pub(crate) description: &'static str, + pub(crate) column_count: usize, + pub(crate) expected_findings: usize, + pub(crate) expected_max_seconds: f64, +} + +pub(crate) struct SafetyRuleTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) rule: &'static str, + pub(crate) expected_anchors: &'static [&'static str], +} diff --git a/crates/sqlquack-lint/src/quality/types.rs b/crates/sqlquack-lint/src/quality/types.rs new file mode 100644 index 000000000..717e03a5e --- /dev/null +++ b/crates/sqlquack-lint/src/quality/types.rs @@ -0,0 +1,6 @@ +use std::collections::BTreeSet; + +/// Column sets that are unique for one relation, by lower-cased output column name. +pub(crate) type Keys = Vec>; +/// A unique column set over a select's joined input, as `(source alias, column)` pairs. +pub(crate) type InputKey = BTreeSet<(String, String)>; diff --git a/crates/sqlquack-lint/src/tree/_helpers/mod.rs b/crates/sqlquack-lint/src/tree/_helpers/mod.rs new file mode 100644 index 000000000..8752239b6 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +//! Helpers. + +pub(crate) mod scope; diff --git a/crates/sqlquack-lint/src/tree/_helpers/scope.rs b/crates/sqlquack-lint/src/tree/_helpers/scope.rs new file mode 100644 index 000000000..8a82b6646 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/_helpers/scope.rs @@ -0,0 +1,123 @@ +//! Tree helpers: which nodes open a scope, and the children a scope walk visits. + +use sqlquack_syntax::expressions::queries::models::{OrderBy, Select}; +use sqlquack_syntax::{Expression, ExpressionWalk}; + +/// Whether `expression` opens a nested scope: a CTE, a derived table, or a query. +pub(crate) fn opens_scope(expression: &Expression) -> bool { + match expression { + Expression::Cte(_) + | Expression::Select(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) => true, + Expression::Subquery(subquery) => subquery.alias.is_some(), + _ => false, + } +} + +/// The children a scope walk visits below `expression`, before nested scopes are dropped. +pub(crate) fn scope_children(expression: &Expression) -> Vec<&Expression> { + match expression { + Expression::Prepare(prepare) => vec![&prepare.statement], + Expression::Select(select) => select_children(select), + Expression::And(operation) + | Expression::Or(operation) + | Expression::Add(operation) + | Expression::Sub(operation) + | Expression::Mul(operation) + | Expression::Div(operation) + | Expression::Mod(operation) + | Expression::Eq(operation) + | Expression::Neq(operation) + | Expression::Lt(operation) + | Expression::Lte(operation) + | Expression::Gt(operation) + | Expression::Gte(operation) + | Expression::BitwiseAnd(operation) + | Expression::BitwiseOr(operation) + | Expression::BitwiseXor(operation) + | Expression::Concat(operation) => vec![&operation.left, &operation.right], + Expression::Like(like) | Expression::ILike(like) => { + let mut children: Vec<&Expression> = vec![&like.left, &like.right]; + children.extend(like.escape.as_ref()); + children + } + Expression::Not(unary) | Expression::Neg(unary) | Expression::BitwiseNot(unary) => { + vec![&unary.this] + } + Expression::Function(function) => function.args.iter().collect(), + Expression::AggregateFunction(function) => function.args.iter().collect(), + Expression::WindowFunction(window) => { + let mut children: Vec<&Expression> = vec![&window.this]; + children.extend(window.over.partition_by.iter()); + children.extend(window.over.order_by.iter().map(|ordered| &ordered.this)); + children + } + Expression::Alias(alias) => vec![&alias.this], + Expression::Case(case) => { + let mut children: Vec<&Expression> = case.operand.iter().collect(); + for (condition, result) in &case.whens { + children.extend([condition, result]); + } + children.extend(case.else_.as_ref()); + children + } + Expression::Paren(paren) => vec![&paren.this], + Expression::Ordered(ordered) => vec![&ordered.this], + Expression::In(membership) => { + let mut children: Vec<&Expression> = vec![&membership.this]; + children.extend(membership.expressions.iter()); + children + } + Expression::Between(between) => vec![&between.this, &between.low, &between.high], + Expression::IsNull(test) => vec![&test.this], + Expression::Cast(cast) => vec![&cast.this], + Expression::Extract(extract) => vec![&extract.this], + Expression::Coalesce(coalesce) => coalesce.expressions.iter().collect(), + Expression::NullIf(nullif) => vec![&nullif.this, &nullif.expression], + Expression::TryCatch(try_catch) => { + let mut children: Vec<&Expression> = try_catch.try_body.iter().collect(); + children.extend(try_catch.catch_body.iter().flatten()); + children + } + Expression::Table(_) + | Expression::Column(_) + | Expression::Literal(_) + | Expression::Identifier(_) + | Expression::Subquery(_) + | Expression::Exists(_) => Vec::new(), + other => other.children(), + } +} + +/// A SELECT's clause expressions, leaving out the relations that open nested scopes. +fn select_children(select: &Select) -> Vec<&Expression> { + let mut children: Vec<&Expression> = select.expressions.iter().collect(); + for relation in select.from.iter().flat_map(|from| from.expressions.iter()) { + children.extend((!opens_scope(relation)).then_some(relation)); + } + for join in &select.joins { + children.extend((!opens_scope(&join.this)).then_some(&join.this)); + children.extend(join.on.as_ref()); + children.extend(join.match_condition.as_ref()); + } + children.extend(select.where_clause.as_ref().map(|clause| &clause.this)); + children.extend( + select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()), + ); + children.extend(select.having.as_ref().map(|having| &having.this)); + children.extend(select.qualify.as_ref().map(|qualify| &qualify.this)); + children.extend(select.order_by.iter().flat_map(ordered_terms)); + children.extend(select.limit.as_ref().map(|limit| &limit.this)); + children.extend(select.offset.as_ref().map(|offset| &offset.this)); + children +} + +/// The expressions an ORDER BY sorts on. +fn ordered_terms(order: &OrderBy) -> impl Iterator { + order.expressions.iter().map(|ordered| &ordered.this) +} diff --git a/crates/sqlquack-lint/src/tree/constants.rs b/crates/sqlquack-lint/src/tree/constants.rs new file mode 100644 index 000000000..f26e61826 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/constants.rs @@ -0,0 +1,4 @@ +//! Constants of tree. + +/// Window-only calls SQLQuack's Snowflake parser keeps as written aggregates; polyglot never counted them. +pub(crate) const WINDOW_ONLY_CALLS: [&str; 2] = ["FIRST_VALUE", "LAST_VALUE"]; diff --git a/crates/sqlquack-lint/src/tree/main/call_name.rs b/crates/sqlquack-lint/src/tree/main/call_name.rs new file mode 100644 index 000000000..034871905 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/call_name.rs @@ -0,0 +1,14 @@ +//! `call_name`: the name of a call the parser kept as written, such as Snowflake's `ROW_NUMBER()`. + +use sqlquack_syntax::Expression; + +/// The name of an unqualified call the parser kept as written; `None` for any other node. +pub(crate) fn call_name(expression: &Expression) -> Option<&str> { + match expression { + Expression::Function(function) if function.qualified_name.is_empty() => { + Some(function.name.as_str()) + } + Expression::AggregateFunction(function) => Some(function.name.as_str()), + _ => None, + } +} diff --git a/crates/sqlquack-lint/src/tree/main/contains_aggregate.rs b/crates/sqlquack-lint/src/tree/main/contains_aggregate.rs new file mode 100644 index 000000000..e235b41c0 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/contains_aggregate.rs @@ -0,0 +1,10 @@ +//! `contains_aggregate`: whether a node or any node below it is an aggregate call. + +use sqlquack_syntax::{Expression, ExpressionWalk}; + +use crate::tree::main::is_aggregate::is_aggregate; + +/// Whether `expression` or any node below it is an aggregate call. +pub(crate) fn contains_aggregate(expression: &Expression) -> bool { + expression.contains(is_aggregate) +} diff --git a/crates/sqlquack-lint/src/tree/main/is_aggregate.rs b/crates/sqlquack-lint/src/tree/main/is_aggregate.rs new file mode 100644 index 000000000..710549eca --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/is_aggregate.rs @@ -0,0 +1,93 @@ +//! `is_aggregate`: whether a node is an aggregate call, as polyglot classified them. + +use sqlquack_syntax::Expression; + +use crate::tree::constants::WINDOW_ONLY_CALLS; + +/// Whether `expression` is an aggregate call; window-only calls and scalar helpers are not. +pub(crate) fn is_aggregate(expression: &Expression) -> bool { + match expression { + Expression::AggregateFunction(function) => !WINDOW_ONLY_CALLS + .iter() + .any(|name| name.eq_ignore_ascii_case(&function.name)), + _ => matches!( + expression, + Expression::Count(_) + | Expression::Sum(_) + | Expression::Avg(_) + | Expression::Min(_) + | Expression::Max(_) + | Expression::GroupConcat(_) + | Expression::StringAgg(_) + | Expression::ListAgg(_) + | Expression::ArrayAgg(_) + | Expression::CountIf(_) + | Expression::SumIf(_) + | Expression::Stddev(_) + | Expression::StddevPop(_) + | Expression::StddevSamp(_) + | Expression::Variance(_) + | Expression::VarPop(_) + | Expression::VarSamp(_) + | Expression::Median(_) + | Expression::Mode(_) + | Expression::First(_) + | Expression::Last(_) + | Expression::AnyValue(_) + | Expression::ApproxDistinct(_) + | Expression::ApproxCountDistinct(_) + | Expression::ApproxPercentile(_) + | Expression::Percentile(_) + | Expression::PercentileCont(_) + | Expression::PercentileDisc(_) + | Expression::LogicalAnd(_) + | Expression::LogicalOr(_) + | Expression::Skewness(_) + | Expression::ArrayConcatAgg(_) + | Expression::ArrayUniqueAgg(_) + | Expression::BoolXorAgg(_) + | Expression::BitwiseAndAgg(_) + | Expression::BitwiseOrAgg(_) + | Expression::BitwiseXorAgg(_) + | Expression::JsonArrayAgg(_) + | Expression::JsonObjectAgg(_) + | Expression::JSONArrayAgg(_) + | Expression::JSONObjectAgg(_) + | Expression::JSONBObjectAgg(_) + | Expression::ParameterizedAgg(_) + | Expression::ArgMax(_) + | Expression::ArgMin(_) + | Expression::ApproxTopK(_) + | Expression::ApproxTopKAccumulate(_) + | Expression::ApproxTopKCombine(_) + | Expression::ApproxTopSum(_) + | Expression::ApproxQuantiles(_) + | Expression::Grouping(_) + | Expression::GroupingId(_) + | Expression::AnonymousAggFunc(_) + | Expression::CombinedAggFunc(_) + | Expression::CombinedParameterizedAgg(_) + | Expression::HashAgg(_) + | Expression::Hll(_) + | Expression::Minhash(_) + | Expression::ObjectAgg(_) + | Expression::AIAgg(_) + | Expression::Quantile(_) + | Expression::ApproxQuantile(_) + | Expression::Corr(_) + | Expression::CovarPop(_) + | Expression::CovarSamp(_) + | Expression::RegrValx(_) + | Expression::RegrValy(_) + | Expression::RegrAvgx(_) + | Expression::RegrAvgy(_) + | Expression::RegrCount(_) + | Expression::RegrIntercept(_) + | Expression::RegrR2(_) + | Expression::RegrSxx(_) + | Expression::RegrSxy(_) + | Expression::RegrSyy(_) + | Expression::RegrSlope(_) + ), + } +} diff --git a/crates/sqlquack-lint/src/tree/main/is_row_number.rs b/crates/sqlquack-lint/src/tree/main/is_row_number.rs new file mode 100644 index 000000000..53dbc89e6 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/is_row_number.rs @@ -0,0 +1,11 @@ +//! `is_row_number`: whether a node is `ROW_NUMBER()`, as a typed node or as written. + +use sqlquack_syntax::Expression; + +use crate::tree::main::call_name::call_name; + +/// Whether `expression` is `ROW_NUMBER()`, as a typed node or as written. +pub(crate) fn is_row_number(expression: &Expression) -> bool { + matches!(expression, Expression::RowNumber(_)) + || call_name(expression).is_some_and(|name| name.eq_ignore_ascii_case("ROW_NUMBER")) +} diff --git a/crates/sqlquack-lint/src/tree/main/mod.rs b/crates/sqlquack-lint/src/tree/main/mod.rs new file mode 100644 index 000000000..d66ffe6b5 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/mod.rs @@ -0,0 +1,7 @@ +//! Main. + +pub(crate) mod call_name; +pub(crate) mod contains_aggregate; +pub(crate) mod is_aggregate; +pub(crate) mod is_row_number; +pub(crate) mod walk_in_scope; diff --git a/crates/sqlquack-lint/src/tree/main/walk_in_scope.rs b/crates/sqlquack-lint/src/tree/main/walk_in_scope.rs new file mode 100644 index 000000000..b1393c6d2 --- /dev/null +++ b/crates/sqlquack-lint/src/tree/main/walk_in_scope.rs @@ -0,0 +1,22 @@ +//! `walk_in_scope`: the nodes of one query scope, depth first, without entering the scopes nested in it. + +use sqlquack_syntax::Expression; + +use crate::tree::_helpers::scope::{opens_scope, scope_children}; + +/// The nodes of the scope `expression` opens, depth first, without entering nested scopes. +pub(crate) fn walk_in_scope(expression: &Expression) -> impl Iterator { + let mut pending: Vec<&Expression> = vec![expression]; + let mut visited: Vec<&Expression> = Vec::new(); + while let Some(node) = pending.pop() { + visited.push(node); + let children: Vec<&Expression> = scope_children(node); + pending.extend( + children + .into_iter() + .rev() + .filter(|child| !opens_scope(child)), + ); + } + visited.into_iter() +} diff --git a/crates/sqlquack-lint/src/tree/mod.rs b/crates/sqlquack-lint/src/tree/mod.rs new file mode 100644 index 000000000..102dada0e --- /dev/null +++ b/crates/sqlquack-lint/src/tree/mod.rs @@ -0,0 +1,5 @@ +//! Tree: scope walks and call classification, ported from polyglot (MIT; see THIRD_PARTY_NOTICES). + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; diff --git a/crates/sqlquack-syntax/src/lib.rs b/crates/sqlquack-syntax/src/lib.rs index d5ace9b26..95dd2d43d 100644 --- a/crates/sqlquack-syntax/src/lib.rs +++ b/crates/sqlquack-syntax/src/lib.rs @@ -27,9 +27,11 @@ pub use crate::modes::models::Gap; pub use crate::modes::models::Parsed; pub use crate::modes::models::TolerantParse; pub use crate::modes::models::TolerantStatement; +pub use crate::parser::main::parse_expressions::parse_expressions; pub use crate::parser::main::parse_statements::parse_statements; pub use crate::parser::main::parse_statements_with::parse_statements_with; pub use crate::tokens::errors::Error; +pub use crate::tokens::main::dialect_tokens::dialect_tokens; pub use crate::tokens::models::CallMode; pub use crate::tokens::models::Dialect; pub use crate::tokens::types::Result; diff --git a/crates/sqlquack-syntax/src/parser/main/mod.rs b/crates/sqlquack-syntax/src/parser/main/mod.rs index c511d22cb..da3ed16bb 100644 --- a/crates/sqlquack-syntax/src/parser/main/mod.rs +++ b/crates/sqlquack-syntax/src/parser/main/mod.rs @@ -1,4 +1,5 @@ //! Main. +pub mod parse_expressions; pub mod parse_statements; pub mod parse_statements_with; diff --git a/crates/sqlquack-syntax/src/parser/main/parse_expressions.rs b/crates/sqlquack-syntax/src/parser/main/parse_expressions.rs new file mode 100644 index 000000000..3e5e0cba6 --- /dev/null +++ b/crates/sqlquack-syntax/src/parser/main/parse_expressions.rs @@ -0,0 +1,18 @@ +//! `parse_expressions`: the comma-separated expressions at the start of `tokens`, ignoring what follows them. + +use crate::expressions::tree::models::Expression; +use crate::parser::models::Parser; +use crate::parser::models::ParserConfig; +use crate::tokens::models::Dialect; +use crate::tokens::models::Token; +use crate::tokens::types::Result; + +/// Leading comma-separated expressions of `tokens`, unguarded: pass only tokens of input already guarded. +pub fn parse_expressions(tokens: Vec, dialect: Dialect) -> Result> { + let config = ParserConfig { + dialect: Some(dialect.dialect_type()), + faithful_calls: true, + ..Default::default() + }; + Parser::with_config(tokens, config).parse_expression_list() +} diff --git a/crates/sqlquack-syntax/src/tokens/main/dialect_tokens.rs b/crates/sqlquack-syntax/src/tokens/main/dialect_tokens.rs new file mode 100644 index 000000000..3e4aaf9a8 --- /dev/null +++ b/crates/sqlquack-syntax/src/tokens/main/dialect_tokens.rs @@ -0,0 +1,12 @@ +//! `dialect_tokens`: the tokens the parser reads for `sql`, each with the comments attached to it. + +use crate::tokens::models::Dialect; +use crate::tokens::models::Token; +use crate::tokens::types::Result; + +/// The tokens the parser reads for `sql`, each with the comments attached to it. +pub fn dialect_tokens(sql: &str, dialect: Dialect) -> Result> { + crate::parser::models::Dialect::get(dialect.dialect_type()) + .tokenizer + .tokenize(sql) +} diff --git a/crates/sqlquack-syntax/src/tokens/main/mod.rs b/crates/sqlquack-syntax/src/tokens/main/mod.rs index 1a9d6a3e9..c9d9b9432 100644 --- a/crates/sqlquack-syntax/src/tokens/main/mod.rs +++ b/crates/sqlquack-syntax/src/tokens/main/mod.rs @@ -1,3 +1,4 @@ //! Main. +pub mod dialect_tokens; pub mod parse_dollar_string_token; From 9284258c3e23cbcc75cc4fbb2c38b3acec0a46ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:06:09 +0000 Subject: [PATCH 22/22] Postgres parser keeps the memo outcome's doc comment on one line --- .../src/parse/_helpers/tree/cursor.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs index 7a91466a6..3a634a4b6 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/cursor.rs @@ -50,8 +50,7 @@ pub(crate) enum Remembered { JsonObjectArguments, } -/// How a remembered construct's parse ended: its result, where it stopped, and the gaps it found. -/// Only failures and `JSON_OBJECT` arguments are kept, so the copies stay bounded. +/// A remembered parse's result, end and gaps; only failures and `JSON_OBJECT` arguments are kept. struct Outcome { result: Parsing, reached: usize, @@ -478,7 +477,7 @@ impl<'sql> Parser<'sql> { result } - /// Drops kept argument trees strictly inside `start..end`; only that span's replay reached them. + /// Drops the kept argument trees inside `start..end`; only that span's replay reached them. fn forget_trees_within(&mut self, start: usize, end: usize) { let inner: Vec<(usize, usize)> = self .kept_trees