From bb0b5cfda4161e6ed2f3c9acac347db8adcf47c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:10:58 +0000 Subject: [PATCH 01/12] feat(format): lay out converted DuckDB SQL with convert --pretty Add sqlquack-format, a whitespace-only formatter over SQLQuack's own tokenizer in SQLClean's documented layout: clauses on their own lines, one select item per line, CTEs separated by a blank line, joins with ON below, CASE and brackets broken only when they do not fit in 100 columns, four-space indent. Token text is never changed; output that would not keep the input's tokens is returned unchanged. - sqlquack convert --pretty lays out the DuckDB SQL; the default output is byte-identical. - The playground engine returns the laid-out SQL, which the playground shows with a Copy button. - A test lays out every distinct converted case statement and checks its tokens and DuckDB parse tree are unchanged. - scripts/measure_format_conformance.py measures agreement with an installed sqlclean format; a measurement, not a gate. --- Cargo.lock | 8 + Cargo.toml | 2 + crates/sqlquack-format/Cargo.toml | 16 ++ .../src/layout/_helpers/clauses.rs | 231 ++++++++++++++++++ .../src/layout/_helpers/expressions.rs | 106 ++++++++ .../src/layout/_helpers/mod.rs | 4 + .../src/layout/_helpers/output.rs | 52 ++++ .../src/layout/_helpers/words.rs | 176 +++++++++++++ .../sqlquack-format/src/layout/constants.rs | 10 + crates/sqlquack-format/src/layout/main/mod.rs | 1 + .../src/layout/main/pretty_sql.rs | 27 ++ crates/sqlquack-format/src/layout/mod.rs | 9 + crates/sqlquack-format/src/layout/models.rs | 75 ++++++ crates/sqlquack-format/src/layout/tests.rs | 4 + .../src/layout/tests/test_pretty_sql.rs | 124 ++++++++++ .../src/layout/tests/test_types.rs | 5 + crates/sqlquack-format/src/lib.rs | 7 + .../src/bindings/_helpers/convert_json.rs | 1 + crates/sqlquack/Cargo.toml | 1 + crates/sqlquack/src/analysis/tests.rs | 2 + crates/sqlquack/src/analysis/tests/helpers.rs | 87 ++++++- .../src/analysis/tests/test_pretty_sql.rs | 19 ++ .../sqlquack/src/analysis/tests/test_types.rs | 5 + .../src/cli/_helpers/convert_command.rs | 7 +- crates/sqlquack/src/cli/models.rs | 3 + scripts/format_conformance/__init__.py | 1 + .../format_conformance/_helpers/__init__.py | 1 + scripts/format_conformance/_helpers/cases.py | 35 +++ .../format_conformance/_helpers/formatters.py | 40 +++ .../format_conformance/_helpers/layouts.py | 49 ++++ scripts/format_conformance/constants.py | 11 + scripts/format_conformance/main/__init__.py | 1 + scripts/format_conformance/main/compare.py | 41 ++++ scripts/format_conformance/types.py | 3 + scripts/measure_format_conformance.py | 14 ++ website/public/playground/playground.js | 10 +- website/public/styles.css | 22 ++ website/src/content/docs/docs/cli/convert.mdx | 1 + website/tests/playground.test.mjs | 7 + website/wasm/Cargo.lock | 8 + website/wasm/Cargo.toml | 1 + website/wasm/src/lib.rs | 2 + 42 files changed, 1226 insertions(+), 3 deletions(-) create mode 100644 crates/sqlquack-format/Cargo.toml create mode 100644 crates/sqlquack-format/src/layout/_helpers/clauses.rs create mode 100644 crates/sqlquack-format/src/layout/_helpers/expressions.rs create mode 100644 crates/sqlquack-format/src/layout/_helpers/mod.rs create mode 100644 crates/sqlquack-format/src/layout/_helpers/output.rs create mode 100644 crates/sqlquack-format/src/layout/_helpers/words.rs create mode 100644 crates/sqlquack-format/src/layout/constants.rs create mode 100644 crates/sqlquack-format/src/layout/main/mod.rs create mode 100644 crates/sqlquack-format/src/layout/main/pretty_sql.rs create mode 100644 crates/sqlquack-format/src/layout/mod.rs create mode 100644 crates/sqlquack-format/src/layout/models.rs create mode 100644 crates/sqlquack-format/src/layout/tests.rs create mode 100644 crates/sqlquack-format/src/layout/tests/test_pretty_sql.rs create mode 100644 crates/sqlquack-format/src/layout/tests/test_types.rs create mode 100644 crates/sqlquack-format/src/lib.rs create mode 100644 crates/sqlquack/src/analysis/tests/test_pretty_sql.rs create mode 100644 scripts/format_conformance/__init__.py create mode 100644 scripts/format_conformance/_helpers/__init__.py create mode 100644 scripts/format_conformance/_helpers/cases.py create mode 100644 scripts/format_conformance/_helpers/formatters.py create mode 100644 scripts/format_conformance/_helpers/layouts.py create mode 100644 scripts/format_conformance/constants.py create mode 100644 scripts/format_conformance/main/__init__.py create mode 100644 scripts/format_conformance/main/compare.py create mode 100644 scripts/format_conformance/types.py create mode 100644 scripts/measure_format_conformance.py diff --git a/Cargo.lock b/Cargo.lock index 149ecdafe..720a0a79f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1354,6 +1354,7 @@ dependencies = [ "sqlquack-core", "sqlquack-duckdb", "sqlquack-evidence", + "sqlquack-format", "sqlquack-policy", "sqlquack-rules", "sqlquack-semantics", @@ -1407,6 +1408,13 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sqlquack-format" +version = "0.2.0" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index b9691a176..a1c0086c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/sqlquack-semantics", "crates/sqlquack-rules", "crates/sqlquack-duckdb", + "crates/sqlquack-format", "crates/sqlquack-policy", "crates/sqlquack-compare", "crates/sqlquack-evidence", @@ -56,6 +57,7 @@ sqlquack-compare = { path = "crates/sqlquack-compare" } sqlquack-core = { path = "crates/sqlquack-core" } sqlquack-duckdb = { path = "crates/sqlquack-duckdb", default-features = false } sqlquack-evidence = { path = "crates/sqlquack-evidence" } +sqlquack-format = { path = "crates/sqlquack-format" } sqlquack-policy = { path = "crates/sqlquack-policy" } sqlquack-rules = { path = "crates/sqlquack-rules" } sqlquack-semantics = { path = "crates/sqlquack-semantics" } diff --git a/crates/sqlquack-format/Cargo.toml b/crates/sqlquack-format/Cargo.toml new file mode 100644 index 000000000..146cc394c --- /dev/null +++ b/crates/sqlquack-format/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "sqlquack-format" +version.workspace = true +description = "Readable layout for SQL: line breaks and indentation between tokens, never a token changed." +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +sqlquack-syntax.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlquack-format/src/layout/_helpers/clauses.rs b/crates/sqlquack-format/src/layout/_helpers/clauses.rs new file mode 100644 index 000000000..4858c08b3 --- /dev/null +++ b/crates/sqlquack-format/src/layout/_helpers/clauses.rs @@ -0,0 +1,231 @@ +//! Laying out statements and queries: each clause starts a line, at its query's indent. + +use sqlquack_syntax::tokens::models::TokenType; + +use crate::layout::_helpers::words::{conditions, items, step, text_is}; +use crate::layout::constants::SELECT_ITEMS_BROKEN; +use crate::layout::models::{Clause, ClauseKind, Printer, Words}; + +impl Printer<'_, '_> { + /// Write every statement, separated by one empty line. + pub(crate) fn statements(&mut self) { + let count = self.words.words.len(); + let (mut start, mut index) = (0, 0); + while index < count { + if self.words.words[index].kind == Some(TokenType::Semicolon) { + self.query(start, index, 0); + self.word(index); + start = index + 1; + if start < count { + self.blank_line(0); + } + } + index = step(self.words, index); + } + self.query(start, count, 0); + } + + /// Write the query `from..to`, its clauses at indent `level`; the first on the current line. + pub(crate) fn query(&mut self, from: usize, to: usize, level: usize) { + let clauses: Vec = clauses(self.words, from, to); + for (position, clause) in clauses.iter().enumerate() { + match position + .checked_sub(1) + .map(|previous| clauses[previous].kind) + { + Some(ClauseKind::With) => self.blank_line(level), + Some(_) => self.newline(level), + None => {} + } + self.flat(clause.start, clause.body); + self.body(clause, level); + } + } + + /// Write what follows a clause's keyword. + fn body(&mut self, clause: &Clause, level: usize) { + let (from, to) = (clause.body, clause.end); + match clause.kind { + ClauseKind::Prefix | ClauseKind::Inline => self.expression(from, to), + ClauseKind::SetOperation => { + if from < to { + self.newline(level); + } + self.expression(from, to); + } + ClauseKind::With => { + let ctes = items(self.words, from, to); + for (position, (start, end, comma)) in ctes.into_iter().enumerate() { + if position > 0 { + self.blank_line(level); + } + self.expression(start, end); + comma.into_iter().for_each(|comma| self.word(comma)); + } + } + ClauseKind::Select | ClauseKind::ListBelow => { + let items = items(self.words, from, to); + let single = clause.kind == ClauseKind::Select && items.len() < SELECT_ITEMS_BROKEN; + if single && (from == to || self.fits(from, to - 1)) { + return self.expression(from, to); + } + for (start, end, comma) in items { + self.newline(level + 1); + self.expression(start, end); + comma.into_iter().for_each(|comma| self.word(comma)); + } + } + ClauseKind::Condition => { + let joins = conditions(self.words, from, to); + if !joins.is_empty() { + self.newline(level + 1); + } + self.condition_lines(from, to, (&joins, level + 1)); + } + ClauseKind::Join => { + let mut on = from; + while on < to && self.words.words[on].kind != Some(TokenType::On) { + on = step(self.words, on); + } + self.expression(from, on); + if on < to { + self.newline(level + 1); + let joins = conditions(self.words, on + 1, to); + self.condition_lines(on, to, (&joins, level + 1)); + } + } + } + } + + /// Write `from..to`, starting a line at `level` before each of its joining `AND` and `OR`. + pub(crate) fn condition_lines( + &mut self, + from: usize, + to: usize, + (joins, level): (&[usize], usize), + ) { + let mut start = from; + for &join in joins { + self.expression(start, join); + self.newline(level); + start = join; + } + self.expression(start, to); + } +} + +/// The clauses of the query `from..to`. +fn clauses(words: &Words<'_>, from: usize, to: usize) -> Vec { + let mut clauses: Vec = Vec::new(); + let mut index = from; + while index < to { + let Some((kind, body)) = clause_at(words, index) else { + index = step(words, index); + continue; + }; + match clauses.last_mut() { + Some(last) => last.end = index, + None if index > from => clauses.push(Clause { + kind: ClauseKind::Prefix, + start: from, + body: from, + end: index, + }), + None => {} + } + let body = body.min(to); + clauses.push(Clause { + kind, + start: index, + body, + end: to, + }); + index = body; + } + if clauses.is_empty() { + clauses.push(Clause { + kind: ClauseKind::Prefix, + start: from, + body: from, + end: to, + }); + } + clauses +} + +/// The clause a keyword at `index` starts, and where its body begins. +fn clause_at(words: &Words<'_>, index: usize) -> Option<(ClauseKind, usize)> { + let kind_of = |index: usize| words.words.get(index).and_then(|word| word.kind); + let kind = kind_of(index)?; + let clause = match kind { + TokenType::Select => { + let mut body = index + 1; + while matches!(kind_of(body), Some(TokenType::Distinct | TokenType::All)) { + body += 1; + } + if kind_of(body) == Some(TokenType::On) && kind_of(body + 1) == Some(TokenType::LParen) + { + body = step(words, body + 1); + } + (ClauseKind::Select, body) + } + TokenType::With => { + let recursive = kind_of(index + 1) == Some(TokenType::Recursive); + (ClauseKind::With, index + 1 + usize::from(recursive)) + } + TokenType::From | TokenType::Limit | TokenType::Offset => (ClauseKind::Inline, index + 1), + TokenType::Where | TokenType::Having | TokenType::Qualify => { + (ClauseKind::Condition, index + 1) + } + TokenType::Group | TokenType::Order if kind_of(index + 1) == Some(TokenType::By) => { + (ClauseKind::ListBelow, index + 2) + } + TokenType::Window => (ClauseKind::ListBelow, index + 1), + TokenType::Union | TokenType::Intersect | TokenType::Except => { + if index + .checked_sub(1) + .is_some_and(|previous| kind_of(previous) == Some(TokenType::Star)) + { + return None; + } + let mut body = index + 1; + if matches!(kind_of(body), Some(TokenType::All | TokenType::Distinct)) { + body += 1; + } + if kind_of(body) == Some(TokenType::By) && text_is(words, body + 1, "NAME") { + body += 2; + } + (ClauseKind::SetOperation, body) + } + _ => { + let mut join = index; + while join_modifier(words, join) { + join += 1; + } + if kind_of(join) != Some(TokenType::Join) { + return None; + } + (ClauseKind::Join, join + 1) + } + }; + Some(clause) +} + +/// Whether the word at `index` can come before `JOIN` in a join's keyword, such as `LEFT`. +fn join_modifier(words: &Words<'_>, index: usize) -> bool { + matches!( + words.words.get(index).and_then(|word| word.kind), + Some( + TokenType::Natural + | TokenType::Left + | TokenType::Right + | TokenType::Full + | TokenType::Inner + | TokenType::Outer + | TokenType::Cross + | TokenType::Semi + | TokenType::Anti + | TokenType::Positional + ) + ) || text_is(words, index, "ASOF") +} diff --git a/crates/sqlquack-format/src/layout/_helpers/expressions.rs b/crates/sqlquack-format/src/layout/_helpers/expressions.rs new file mode 100644 index 000000000..ddfaf77d3 --- /dev/null +++ b/crates/sqlquack-format/src/layout/_helpers/expressions.rs @@ -0,0 +1,106 @@ +//! Laying out expressions: brackets that fit stay on their line, others break, outermost first. + +use sqlquack_syntax::tokens::models::TokenType; + +use crate::layout::_helpers::words::{ + conditions, forced, items, kind_at, opens_query, spaced, step, width, window_orders, +}; +use crate::layout::constants::LINE_WIDTH; +use crate::layout::models::Printer; + +impl Printer<'_, '_> { + /// Write `from..to`, breaking each bracket that holds a query or CASE, or does not fit. + pub(crate) fn expression(&mut self, from: usize, to: usize) { + let mut index = from; + while index < to { + let next = step(self.words, index); + let closes_inside = next <= to && next - 1 > index; + match self.words.words[index].kind { + Some(TokenType::LParen) if closes_inside => self.bracket(index, next - 1), + Some(TokenType::Case) if closes_inside => self.case(index, next - 1), + _ => self.word(index), + } + index = next; + } + } + + /// Whether `from..=to` fits on the current line and holds nothing that must break. + pub(crate) fn fits(&self, from: usize, to: usize) -> bool { + let space = usize::from(!self.output.line_start && spaced(self.words, from)); + let width = self.output.column + space + width(self.words, from, to); + width <= LINE_WIDTH && !forced(self.words, from, to) + } + + /// Write the bracket from `open` to `close`: on the line when it fits, else over lines. + fn bracket(&mut self, open: usize, close: usize) { + let level = self.output.level; + if self.fits(open, close) { + return self.flat(open, close + 1); + } + self.word(open); + let case = kind_at(self.words, open + 1) == Some(TokenType::Case); + if case && step(self.words, open + 1) == close { + self.case(open + 1, close - 1); + return self.word(close); + } + let window = matches!( + kind_at(self.words, open + 1), + Some(TokenType::Partition | TokenType::Order) + ); + match (opens_query(self.words, open), window) { + (true, _) => { + self.newline(level + 1); + self.query(open + 1, close, level + 1); + } + (false, true) => { + self.newline(level + 1); + let orders = window_orders(self.words, open + 1, close); + self.condition_lines(open + 1, close, (&orders, level + 1)); + } + (false, false) => { + for (start, end, comma) in items(self.words, open + 1, close) { + self.newline(level + 1); + self.expression(start, end); + comma.into_iter().for_each(|comma| self.word(comma)); + } + } + } + self.newline(level); + self.word(close); + } + + /// Write the CASE from `case` to `end`: on the line when it fits, else one branch per line. + fn case(&mut self, case: usize, end: usize) { + if self.fits(case, end) { + return self.flat(case, end + 1); + } + let level = self.output.level; + self.word(case); + let mut start = case + 1; + let mut index = start; + while index < end { + let branch = matches!( + self.words.words[index].kind, + Some(TokenType::When | TokenType::Then | TokenType::Else) + ); + if branch { + self.branch(start, index, level + 1); + self.newline(level + 1); + start = index; + } + index = step(self.words, index); + } + self.branch(start, end, level + 1); + self.newline(level); + self.word(end); + } + + /// Write a CASE branch `from..to`, breaking before its `AND` and `OR` when it is too wide. + fn branch(&mut self, from: usize, to: usize, level: usize) { + let joins = match from < to && self.fits(from, to - 1) { + true => Vec::new(), + false => conditions(self.words, from + 1, to), + }; + self.condition_lines(from, to, (&joins, level)); + } +} diff --git a/crates/sqlquack-format/src/layout/_helpers/mod.rs b/crates/sqlquack-format/src/layout/_helpers/mod.rs new file mode 100644 index 000000000..03df80ad5 --- /dev/null +++ b/crates/sqlquack-format/src/layout/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod clauses; +pub(crate) mod expressions; +pub(crate) mod output; +pub(crate) mod words; diff --git a/crates/sqlquack-format/src/layout/_helpers/output.rs b/crates/sqlquack-format/src/layout/_helpers/output.rs new file mode 100644 index 000000000..6337daae1 --- /dev/null +++ b/crates/sqlquack-format/src/layout/_helpers/output.rs @@ -0,0 +1,52 @@ +//! Writing words and line breaks to the formatted text. + +use crate::layout::_helpers::words::spaced; +use crate::layout::constants::INDENT_WIDTH; +use crate::layout::models::Printer; + +impl Printer<'_, '_> { + /// Start a new line at indent `level`. + pub(crate) fn newline(&mut self, level: usize) { + let output = &mut self.output; + output.level = level; + output.line_start = true; + if output.text.is_empty() { + return; + } + let kept = output.text.trim_end_matches(' ').len(); + output.text.truncate(kept); + output.text.push('\n'); + output.text.push_str(&" ".repeat(level * INDENT_WIDTH)); + output.column = level * INDENT_WIDTH; + output.after_line_comment = false; + } + + /// Leave one empty line, then start a line at indent `level`. + pub(crate) fn blank_line(&mut self, level: usize) { + self.newline(0); + self.newline(level); + } + + /// Write the word at `index` on the current line, after a space where it takes one. + pub(crate) fn word(&mut self, index: usize) { + if self.output.after_line_comment { + self.newline(self.output.level); + } + let space = spaced(self.words, index); + let word = &self.words.words[index]; + let output = &mut self.output; + if !output.line_start && !output.text.is_empty() && space { + output.text.push(' '); + output.column += 1; + } + output.text.push_str(word.text); + output.column += word.text.chars().count(); + output.line_start = false; + output.after_line_comment = word.line_comment; + } + + /// Write the words `from..to` on the current line, as they were spaced. + pub(crate) fn flat(&mut self, from: usize, to: usize) { + (from..to).for_each(|index| self.word(index)); + } +} diff --git a/crates/sqlquack-format/src/layout/_helpers/words.rs b/crates/sqlquack-format/src/layout/_helpers/words.rs new file mode 100644 index 000000000..94895a130 --- /dev/null +++ b/crates/sqlquack-format/src/layout/_helpers/words.rs @@ -0,0 +1,176 @@ +//! The words of the SQL, and how its brackets, lists and conditions divide them. + +use sqlquack_syntax::lossless::models::{LosslessToken, TokenKind}; +use sqlquack_syntax::tokens::models::TokenType; +use sqlquack_syntax::{Dialect, tokenize}; + +use crate::layout::models::{Word, Words}; + +/// Every token but whitespace, whether whitespace preceded it, and where each bracket closes. +pub(crate) fn words<'sql>(sql: &'sql str, tokens: &[LosslessToken]) -> Words<'sql> { + let mut words: Vec> = Vec::with_capacity(tokens.len()); + let mut spaced = false; + for token in tokens { + let (kind, line_comment) = match token.kind { + TokenKind::Whitespace => { + spaced = true; + continue; + } + TokenKind::Significant(kind) => (Some(kind), false), + TokenKind::LineComment => (None, true), + TokenKind::BlockComment => (None, false), + }; + words.push(Word { + text: token.text(sql), + kind, + line_comment, + spaced, + }); + spaced = false; + } + let mut closer: Vec> = vec![None; words.len()]; + let (mut brackets, mut cases): (Vec, Vec) = (Vec::new(), Vec::new()); + for (index, word) in words.iter().enumerate() { + match word.kind { + Some(TokenType::LParen) => brackets.push(index), + Some(TokenType::Case) => cases.push(index), + Some(TokenType::RParen) => { + if let Some(open) = brackets.pop() { + closer[open] = Some(index); + } + } + Some(TokenType::End) => { + if let Some(open) = cases.pop() { + closer[open] = Some(index); + } + } + _ => {} + } + } + Words { words, closer } +} + +/// The word after `index` at the same level: past the bracket or CASE that opens at `index`. +pub(crate) fn step(words: &Words<'_>, index: usize) -> usize { + words.closer.get(index).copied().flatten().unwrap_or(index) + 1 +} + +/// The kind of the first word from `index` that is not a comment. +pub(crate) fn kind_at(words: &Words<'_>, index: usize) -> Option { + words + .words + .get(index..) + .unwrap_or_default() + .iter() + .find_map(|word| word.kind) +} + +/// Whether `text` is the word at `index`, ignoring case. +pub(crate) fn text_is(words: &Words<'_>, index: usize, text: &str) -> bool { + words + .words + .get(index) + .is_some_and(|word| word.text.eq_ignore_ascii_case(text)) +} + +/// Whether the bracket at `open` holds a query, itself perhaps in brackets. +pub(crate) fn opens_query(words: &Words<'_>, open: usize) -> bool { + match kind_at(words, open + 1) { + Some(TokenType::Select | TokenType::With) => true, + Some(TokenType::LParen) => opens_query(words, open + 1), + _ => false, + } +} + +/// The ranges of `from..to` between its commas at this level, and each comma after a range. +pub(crate) fn items( + words: &Words<'_>, + from: usize, + to: usize, +) -> Vec<(usize, usize, Option)> { + let mut items: Vec<(usize, usize, Option)> = Vec::new(); + let (mut start, mut index) = (from, from); + while index < to { + if words.words[index].kind == Some(TokenType::Comma) { + items.push((start, index, Some(index))); + start = index + 1; + } + index = step(words, index); + } + if start < to { + items.push((start, to, None)); + } + items +} + +/// Each `AND` or `OR` joining two conditions of `from..to` at this level, not `BETWEEN`'s `AND`. +pub(crate) fn conditions(words: &Words<'_>, from: usize, to: usize) -> Vec { + let mut joins: Vec = Vec::new(); + let mut between = false; + let mut index = from; + while index < to { + match words.words[index].kind { + Some(TokenType::Between) => between = true, + Some(TokenType::And) if between => between = false, + Some(TokenType::And | TokenType::Or) if index > from => joins.push(index), + _ => {} + } + index = step(words, index); + } + joins +} + +/// Whether `from..=to` must break over lines: it holds a query or a line comment. +pub(crate) fn forced(words: &Words<'_>, from: usize, to: usize) -> bool { + (from..=to).any(|index| { + let word = &words.words[index]; + word.line_comment || (word.kind == Some(TokenType::LParen) && opens_query(words, index)) + }) +} + +/// How wide `from..=to` is on one line, with a space after every comma. +pub(crate) fn width(words: &Words<'_>, from: usize, to: usize) -> usize { + (from..=to) + .map(|index| { + let spaced = index > from && spaced(words, index); + words.words[index].text.chars().count() + usize::from(spaced) + }) + .sum() +} + +/// Whether the word at `index` takes a space on a shared line: after whitespace, a comma or `/`. +pub(crate) fn spaced(words: &Words<'_>, index: usize) -> bool { + let word = &words.words[index]; + let previous = index + .checked_sub(1) + .and_then(|previous| words.words[previous].kind); + word.spaced + || word.kind == Some(TokenType::Slash) + || matches!(previous, Some(TokenType::Comma | TokenType::Slash)) +} + +/// Whether `sql` has exactly the words `expected` holds. +pub(crate) fn same_words(sql: &str, dialect: Dialect, expected: &Words<'_>) -> bool { + let Ok(tokens) = tokenize(sql, dialect) else { + return false; + }; + let found = words(sql, &tokens); + found.words.len() == expected.words.len() + && found + .words + .iter() + .zip(&expected.words) + .all(|(found, expected)| found.text == expected.text && found.kind == expected.kind) +} + +/// Where a window specification `from..to` starts its `ORDER BY`, after a `PARTITION BY`. +pub(crate) fn window_orders(words: &Words<'_>, from: usize, to: usize) -> Vec { + let mut index = from + 1; + while index < to { + if words.words[index].kind == Some(TokenType::Order) { + return vec![index]; + } + index = step(words, index); + } + Vec::new() +} diff --git a/crates/sqlquack-format/src/layout/constants.rs b/crates/sqlquack-format/src/layout/constants.rs new file mode 100644 index 000000000..73f0fbc7e --- /dev/null +++ b/crates/sqlquack-format/src/layout/constants.rs @@ -0,0 +1,10 @@ +//! Layout constants, SQLClean's defaults. + +/// Spaces per indent step. +pub const INDENT_WIDTH: usize = 4; + +/// The width a line fills before a bracket on it is broken over lines. +pub const LINE_WIDTH: usize = 100; + +/// A SELECT list of at least this many items puts one item per line. +pub const SELECT_ITEMS_BROKEN: usize = 2; diff --git a/crates/sqlquack-format/src/layout/main/mod.rs b/crates/sqlquack-format/src/layout/main/mod.rs new file mode 100644 index 000000000..e252a05b5 --- /dev/null +++ b/crates/sqlquack-format/src/layout/main/mod.rs @@ -0,0 +1 @@ +pub mod pretty_sql; diff --git a/crates/sqlquack-format/src/layout/main/pretty_sql.rs b/crates/sqlquack-format/src/layout/main/pretty_sql.rs new file mode 100644 index 000000000..6f7c50aed --- /dev/null +++ b/crates/sqlquack-format/src/layout/main/pretty_sql.rs @@ -0,0 +1,27 @@ +//! `pretty_sql`: SQL laid out over lines, with exactly the tokens it had. + +use sqlquack_syntax::{Dialect, tokenize}; + +use crate::layout::_helpers::words::{same_words, words}; +use crate::layout::models::{Output, Printer}; + +/// `sql` laid out over lines with its tokens unchanged; returned as it is when that cannot be done. +pub fn pretty_sql(sql: &str, dialect: Dialect) -> String { + let Ok(tokens) = tokenize(sql, dialect) else { + return sql.to_string(); + }; + let words = words(sql, &tokens); + let mut printer = Printer { + words: &words, + output: Output { + line_start: true, + ..Output::default() + }, + }; + printer.statements(); + let text: String = printer.output.text; + match same_words(&text, dialect, &words) { + true => text, + false => sql.to_string(), + } +} diff --git a/crates/sqlquack-format/src/layout/mod.rs b/crates/sqlquack-format/src/layout/mod.rs new file mode 100644 index 000000000..de2c54241 --- /dev/null +++ b/crates/sqlquack-format/src/layout/mod.rs @@ -0,0 +1,9 @@ +//! Layout: clauses on their own lines, lists one item per line, nested queries indented. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-format/src/layout/models.rs b/crates/sqlquack-format/src/layout/models.rs new file mode 100644 index 000000000..8f1a74303 --- /dev/null +++ b/crates/sqlquack-format/src/layout/models.rs @@ -0,0 +1,75 @@ +//! The SQL being laid out, and the text laid out so far. + +use sqlquack_syntax::tokens::models::TokenType; + +/// One token of the SQL, with whether whitespace preceded it. +#[derive(Debug, Clone)] +pub(crate) struct Word<'sql> { + /// The token's exact text. + pub(crate) text: &'sql str, + /// The significant token type; None for a comment. + pub(crate) kind: Option, + /// Whether a line comment, which must end its line. + pub(crate) line_comment: bool, + /// Whether whitespace separated it from the token before. + pub(crate) spaced: bool, +} + +/// The tokens of the SQL, and where each bracket or CASE closes. +#[derive(Debug)] +pub(crate) struct Words<'sql> { + pub(crate) words: Vec>, + /// For a `(` its `)`, for a CASE its END. + pub(crate) closer: Vec>, +} + +/// The words being laid out, and the text laid out so far. +#[derive(Debug)] +pub(crate) struct Printer<'w, 'sql> { + pub(crate) words: &'w Words<'sql>, + pub(crate) output: Output, +} + +/// The formatted text so far. +#[derive(Debug, Default)] +pub(crate) struct Output { + pub(crate) text: String, + /// Characters on the current line. + pub(crate) column: usize, + /// The indent level of the current line. + pub(crate) level: usize, + /// Nothing but indentation on the current line yet. + pub(crate) line_start: bool, + /// The last word was a line comment, so the next word starts a line. + pub(crate) after_line_comment: bool, +} + +/// How a query clause lays out what follows its keyword. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ClauseKind { + /// Words before the first clause keyword, such as `CREATE TABLE t AS`. + Prefix, + /// `WITH`: common table expressions, separated by a blank line. + With, + /// `SELECT`: one item per line when there are two or more. + Select, + /// `GROUP BY`, `ORDER BY`, `WINDOW`: items on the lines below. + ListBelow, + /// `WHERE`, `HAVING`, `QUALIFY`: one condition on the line, several on the lines below. + Condition, + /// A join: the relation on the line, `ON` on the next. + Join, + /// `UNION`, `INTERSECT`, `EXCEPT`, on a line of their own. + SetOperation, + /// `FROM`, `LIMIT` and the rest: on the keyword's line. + Inline, +} + +/// One clause of a query: its keyword words, then its body up to the next clause. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Clause { + pub(crate) kind: ClauseKind, + pub(crate) start: usize, + pub(crate) body: usize, + pub(crate) end: usize, +} diff --git a/crates/sqlquack-format/src/layout/tests.rs b/crates/sqlquack-format/src/layout/tests.rs new file mode 100644 index 000000000..5ba425d5e --- /dev/null +++ b/crates/sqlquack-format/src/layout/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_pretty_sql.rs"] +mod test_pretty_sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-format/src/layout/tests/test_pretty_sql.rs b/crates/sqlquack-format/src/layout/tests/test_pretty_sql.rs new file mode 100644 index 000000000..fb47f6e56 --- /dev/null +++ b/crates/sqlquack-format/src/layout/tests/test_pretty_sql.rs @@ -0,0 +1,124 @@ +use sqlquack_syntax::Dialect; + +use crate::layout::tests::test_types::PrettySqlTestCase; +use crate::pretty_sql; + +#[test] +fn given_converted_sql_when_laid_out_then_it_takes_sqlcleans_layout() { + let test_cases = [ + PrettySqlTestCase { + description: "select items, CASE and GROUP BY and ORDER BY lists break onto lines", + sql: r#"SELECT "ORDERS"."CUSTOMER_ID" AS "CUSTOMER_ID", CAST(sum("ORDERS"."TOTAL") AS DECIMAL(24,2)) AS "REVENUE", (CASE WHEN (CAST(count(*) AS DECIMAL(18,0)) > CAST(1 AS DECIMAL(1,0))) THEN CAST('repeat' AS VARCHAR) ELSE CAST('new' AS VARCHAR) END) AS "SEGMENT" FROM "ORDERS" AS "ORDERS" WHERE ("ORDERS"."STATUS" = 'shipped') GROUP BY "ORDERS"."CUSTOMER_ID" ORDER BY CAST(sum("ORDERS"."TOTAL") AS DECIMAL(24,2)) DESC NULLS FIRST"#, + expected_sql: r#"SELECT + "ORDERS"."CUSTOMER_ID" AS "CUSTOMER_ID", + CAST(sum("ORDERS"."TOTAL") AS DECIMAL(24, 2)) AS "REVENUE", + (CASE + WHEN (CAST(count(*) AS DECIMAL(18, 0)) > CAST(1 AS DECIMAL(1, 0))) + THEN CAST('repeat' AS VARCHAR) + ELSE CAST('new' AS VARCHAR) + END) AS "SEGMENT" +FROM "ORDERS" AS "ORDERS" +WHERE ("ORDERS"."STATUS" = 'shipped') +GROUP BY + "ORDERS"."CUSTOMER_ID" +ORDER BY + CAST(sum("ORDERS"."TOTAL") AS DECIMAL(24, 2)) DESC NULLS FIRST"#, + }, + PrettySqlTestCase { + description: "CTEs, joins and several conditions take their own lines", + sql: r#"WITH "a" AS (SELECT "x"."id" AS "id", "x"."v" AS "v" FROM "x" AS "x" WHERE (("x"."v" > 1) AND ("x"."id" < 9))), "b" AS (SELECT 1 AS "one") SELECT "a"."id" AS "id" FROM "a" AS "a" LEFT JOIN "b" AS "b" ON ("a"."id" = "b"."one") INNER JOIN "c" ON "a"."id" = "c"."id" AND "c"."k" = 2 WHERE "a"."id" IN (SELECT "id" FROM "z") AND "a"."v" BETWEEN 2 AND 5 QUALIFY row_number() OVER (PARTITION BY "a"."id" ORDER BY "a"."v" DESC) = 1 LIMIT 10"#, + expected_sql: r#"WITH "a" AS ( + SELECT + "x"."id" AS "id", + "x"."v" AS "v" + FROM "x" AS "x" + WHERE (("x"."v" > 1) AND ("x"."id" < 9)) +), + +"b" AS ( + SELECT 1 AS "one" +) + +SELECT "a"."id" AS "id" +FROM "a" AS "a" +LEFT JOIN "b" AS "b" + ON ("a"."id" = "b"."one") +INNER JOIN "c" + ON "a"."id" = "c"."id" + AND "c"."k" = 2 +WHERE + "a"."id" IN ( + SELECT "id" + FROM "z" + ) + AND "a"."v" BETWEEN 2 AND 5 +QUALIFY row_number() OVER (PARTITION BY "a"."id" ORDER BY "a"."v" DESC) = 1 +LIMIT 10"#, + }, + PrettySqlTestCase { + description: "a wide call wraps one argument per line, and set operations and statements split queries", + sql: r#"SELECT coalesce("t"."aaaaaaaaaaaaaaaa", "t"."bbbbbbbbbbbbbbbbbbbb", "t"."cccccccccccccccccccccc", "t"."dddddddddddddddd") AS "x", 1 AS "y" FROM "t" AS "t" UNION ALL SELECT 2, 3 FROM "u"; SELECT 1"#, + expected_sql: r#"SELECT + coalesce( + "t"."aaaaaaaaaaaaaaaa", + "t"."bbbbbbbbbbbbbbbbbbbb", + "t"."cccccccccccccccccccccc", + "t"."dddddddddddddddd" + ) AS "x", + 1 AS "y" +FROM "t" AS "t" +UNION ALL +SELECT + 2, + 3 +FROM "u"; + +SELECT 1"#, + }, + PrettySqlTestCase { + description: "comments and the text inside strings are kept", + sql: "SELECT 'a, b' AS \"x\", -- note\n 2 AS \"y\" /* c */ FROM \"t\"", + expected_sql: "SELECT\n 'a, b' AS \"x\",\n -- note\n 2 AS \"y\" /* c */\nFROM \"t\"", + }, + PrettySqlTestCase { + description: "bracketed set operations take their own lines, and division is spaced", + sql: r#"SELECT 500/1000 AS "A", (SELECT 1 FROM "x") AS "B" FROM ((SELECT 1 AS "K" FROM "a") UNION ALL (SELECT 2 AS "K" FROM "b")) AS "T""#, + expected_sql: r#"SELECT + 500 / 1000 AS "A", + ( + SELECT 1 + FROM "x" + ) AS "B" +FROM ( + ( + SELECT 1 AS "K" + FROM "a" + ) + UNION ALL + ( + SELECT 2 AS "K" + FROM "b" + ) +) AS "T""#, + }, + PrettySqlTestCase { + description: "a wide single select item goes below SELECT, its window one clause per line", + sql: r#"SELECT LAG("INPUT"."AMOUNT", 2 IGNORE NULLS) OVER (PARTITION BY "INPUT"."CATEGORY" ORDER BY "INPUT"."SEQ" ASC NULLS LAST) AS "LAG_OFFSET_IGNORE_NULLS" FROM "INPUT" AS "INPUT""#, + expected_sql: r#"SELECT + LAG("INPUT"."AMOUNT", 2 IGNORE NULLS) OVER ( + PARTITION BY "INPUT"."CATEGORY" + ORDER BY "INPUT"."SEQ" ASC NULLS LAST + ) AS "LAG_OFFSET_IGNORE_NULLS" +FROM "INPUT" AS "INPUT""#, + }, + ]; + + for test_case in test_cases { + assert_eq!( + pretty_sql(test_case.sql, Dialect::DuckDB), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-format/src/layout/tests/test_types.rs b/crates/sqlquack-format/src/layout/tests/test_types.rs new file mode 100644 index 000000000..a26a46e44 --- /dev/null +++ b/crates/sqlquack-format/src/layout/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct PrettySqlTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_sql: &'static str, +} diff --git a/crates/sqlquack-format/src/lib.rs b/crates/sqlquack-format/src/lib.rs new file mode 100644 index 000000000..d13c9295b --- /dev/null +++ b/crates/sqlquack-format/src/lib.rs @@ -0,0 +1,7 @@ +//! Readable layout for SQL that changes only the whitespace between its tokens. + +#![forbid(unsafe_code)] + +pub mod layout; + +pub use crate::layout::main::pretty_sql::pretty_sql; diff --git a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs index b265a356f..fa86d50c0 100644 --- a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs +++ b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs @@ -21,6 +21,7 @@ pub(crate) fn convert_json(sql: &str, request: &str, path: &str) -> PyResult DivergenceModeOutco categories: vec![], } } + +/// Each repository case whose converted SQL loses a token or its DuckDB parse tree laid out. +pub(crate) fn pretty_layout_mismatches() -> Vec { + let root = repository_root(); + let policy = load_policy(&root.join(POLICY_FILE_NAME)).expect("the repository policy loads"); + let cases = load_cases(&root.join(CASES_DIR), &policy).expect("the repository cases load"); + let converter = PipelineConverter { + options: conversion_options(&policy, false), + python: None, + }; + let converted: BTreeMap = cases + .iter() + .filter_map(|case| { + Some(( + converted_text(converter.convert(case))?, + case.reference.clone(), + )) + }) + .rev() + .collect(); + assert!( + converted.len() > 100, + "{} statements converted", + converted.len() + ); + let connection = duckdb::Connection::open_in_memory().expect("DuckDB opens"); + converted + .iter() + .filter(|(sql, _)| { + let pretty = pretty_sql(sql, Dialect::DuckDB); + let tree = parse_tree(&connection, sql); + !tree.starts_with(PARSED_TREE) + || significant_tokens(&pretty) != significant_tokens(sql) + || parse_tree(&connection, &pretty) != tree + || (pretty == **sql && sql.contains(" FROM ")) + }) + .map(|(_, reference)| reference.clone()) + .collect() +} + +/// The SQL of a conversion, or `None` when it refused. +fn converted_text(conversion: Conversion) -> Option { + let Conversion::Converted { sql, .. } = conversion else { + return None; + }; + Some(sql) +} + +/// The text of each token of DuckDB SQL but whitespace. +fn significant_tokens(sql: &str) -> Vec<&str> { + tokenize(sql, Dialect::DuckDB) + .expect("converted SQL tokenizes") + .iter() + .filter(|token| token.kind != TokenKind::Whitespace) + .map(|token| token.text(sql)) + .collect() +} + +/// DuckDB's parse tree of `sql` as JSON, without the source offsets that whitespace moves. +fn parse_tree(connection: &duckdb::Connection, sql: &str) -> String { + let tree: String = connection + .query_row("SELECT json_serialize_sql(?)", [sql], |row| row.get(0)) + .expect("DuckDB serializes the parse tree"); + let mut parts = tree.split(LOCATION_FIELD); + let first: &str = parts.next().unwrap_or_default(); + std::iter::once(first) + .chain(parts.map(|part| { + part.trim_start_matches(location_rest) + .trim_start_matches(',') + })) + .collect() +} + +/// Whether `character` can follow `"query_location` in its field: `_length`, `":` and digits. +fn location_rest(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '_' | '"' | ':') +} + +/// How DuckDB's serialized parse tree of SQL that parses starts. +const PARSED_TREE: &str = r#"{"error":false"#; + +/// The start of DuckDB's source-offset fields, `query_location` and `query_location_length`. +const LOCATION_FIELD: &str = r#""query_location"#; diff --git a/crates/sqlquack/src/analysis/tests/test_pretty_sql.rs b/crates/sqlquack/src/analysis/tests/test_pretty_sql.rs new file mode 100644 index 000000000..7f0c40729 --- /dev/null +++ b/crates/sqlquack/src/analysis/tests/test_pretty_sql.rs @@ -0,0 +1,19 @@ +use crate::analysis::tests::helpers::pretty_layout_mismatches; +use crate::analysis::tests::test_types::PrettyLayoutTestCase; + +#[test] +fn given_repository_cases_when_converted_sql_is_laid_out_then_tokens_and_parse_trees_are_kept() { + let test_cases = [PrettyLayoutTestCase { + description: "every distinct converted statement is laid out with the same tokens and DuckDB parse tree", + expected_mismatches: Vec::new(), + }]; + + for test_case in test_cases { + assert_eq!( + pretty_layout_mismatches(), + test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/analysis/tests/test_types.rs b/crates/sqlquack/src/analysis/tests/test_types.rs index e64bb1af6..d97677dc6 100644 --- a/crates/sqlquack/src/analysis/tests/test_types.rs +++ b/crates/sqlquack/src/analysis/tests/test_types.rs @@ -20,6 +20,11 @@ pub(crate) struct EquivalenceCasesTestCase { pub(crate) expected_mismatches: Vec, } +pub(crate) struct PrettyLayoutTestCase { + pub(crate) description: &'static str, + pub(crate) expected_mismatches: Vec, +} + pub(crate) struct DivergenceModeTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/crates/sqlquack/src/cli/_helpers/convert_command.rs b/crates/sqlquack/src/cli/_helpers/convert_command.rs index fda0b0e3b..b747fc587 100644 --- a/crates/sqlquack/src/cli/_helpers/convert_command.rs +++ b/crates/sqlquack/src/cli/_helpers/convert_command.rs @@ -12,10 +12,12 @@ use sqlquack_core::diagnostics::main::unplaced_diagnostic::unplaced_diagnostic; use sqlquack_core::diagnostics::models::Diagnostic; use sqlquack_core::provenance::main::merge_provenance::merge_provenance; use sqlquack_core::refusals::models::Refusal; +use sqlquack_format::pretty_sql; use sqlquack_policy::policy::constants::{SESSION_TIME_ZONE, TIMEZONE_PARAMETER}; use sqlquack_policy::policy::models::{DivergenceFailure, Policy}; use sqlquack_rules::conversion::models::ConversionOptions; use sqlquack_semantics::types::models::Schema; +use sqlquack_syntax::Dialect; use std::collections::{BTreeMap, BTreeSet}; use crate::analysis::main::assumption_id::assumption_id; @@ -53,7 +55,10 @@ pub(crate) fn convert_command( return ExitCode::from(2); } }; - let (report, sql) = report; + let (mut report, sql) = report; + if arguments.pretty { + report.sql = pretty_sql(&report.sql, Dialect::DuckDB); + } let rendered = match arguments.output.json() { true => serde_json::to_string_pretty(&report).unwrap_or_default() + "\n", false if report.converted => { diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs index c727b54f9..4435b8f8d 100644 --- a/crates/sqlquack/src/cli/models.rs +++ b/crates/sqlquack/src/cli/models.rs @@ -103,6 +103,9 @@ pub struct ConvertArguments { /// A Snowflake session parameter of the caller's session, `NAME=VALUE`; repeatable. #[arg(long = "session", value_name = "NAME=VALUE")] pub session: Vec, + /// Lay the DuckDB SQL out over lines, one clause per line; only whitespace changes. + #[arg(long)] + pub pretty: bool, /// How to print the report on standard output. #[command(flatten)] pub output: OutputArguments, diff --git a/scripts/format_conformance/__init__.py b/scripts/format_conformance/__init__.py new file mode 100644 index 000000000..cb7e599f7 --- /dev/null +++ b/scripts/format_conformance/__init__.py @@ -0,0 +1 @@ +"""SQLQuack formatter conformance with SQLClean.""" diff --git a/scripts/format_conformance/_helpers/__init__.py b/scripts/format_conformance/_helpers/__init__.py new file mode 100644 index 000000000..e37e03b6b --- /dev/null +++ b/scripts/format_conformance/_helpers/__init__.py @@ -0,0 +1 @@ +"""Helpers.""" diff --git a/scripts/format_conformance/_helpers/cases.py b/scripts/format_conformance/_helpers/cases.py new file mode 100644 index 000000000..8e6ca9536 --- /dev/null +++ b/scripts/format_conformance/_helpers/cases.py @@ -0,0 +1,35 @@ +"""The repository cases' queries, each with a schema declaring its input relation.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from scripts.format_conformance.constants import ( + COLUMNS_KEY, + INPUT_RELATION, + QUERY_KEY, + QUOTE, +) +from scripts.format_conformance.types import Schema + + +def case_queries(cases_dir: Path) -> list[tuple[str, str, Schema]]: + """Every case's id, Snowflake query and schema, in file order.""" + queries: list[tuple[str, str, Schema]] = [] + for path in sorted(cases_dir.rglob("*.toml")): + document: dict = tomllib.loads(path.read_text()) + for case in document.get("case", []): + if not {QUERY_KEY, COLUMNS_KEY} <= case.keys(): + continue + columns: list[list[str]] = [ + [column_name(column["name"]), column["type"]] for column in case[COLUMNS_KEY] + ] + queries.append((case["id"], case[QUERY_KEY], {INPUT_RELATION: columns})) + return queries + + +def column_name(name: str) -> str: + """A column name as Snowflake resolves it: quoted names as written, others upper case.""" + quoted: bool = len(name) > 1 and name[0] == name[-1] == QUOTE + return name[1:-1] if quoted else name.upper() diff --git a/scripts/format_conformance/_helpers/formatters.py b/scripts/format_conformance/_helpers/formatters.py new file mode 100644 index 000000000..2c17bed08 --- /dev/null +++ b/scripts/format_conformance/_helpers/formatters.py @@ -0,0 +1,40 @@ +"""Running `sqlquack convert`, with and without `--pretty`, and `sqlclean format`.""" + +from __future__ import annotations + +import json +import subprocess +import tempfile +from pathlib import Path + +from scripts.format_conformance.constants import SQLCLEAN_CONFIG + + +def converted(*, sqlquack: str, query: str, schema: dict) -> tuple[str, str] | None: + """The DuckDB SQL `sqlquack convert` prints for `query`, plain and with `--pretty`.""" + with tempfile.NamedTemporaryFile("w", suffix=".json") as schema_file: + json.dump(schema, schema_file) + schema_file.flush() + outputs: list[str] = [] + for flags in ([], ["--pretty"]): + command: list[str] = [sqlquack, "convert", "--utc-session", "--format", "json"] + command += ["--schema", schema_file.name, *flags, "-"] + result: subprocess.CompletedProcess[str] = subprocess.run( + command, input=query, capture_output=True, text=True, check=False + ) + if result.returncode != 0: + return None + outputs.append(json.loads(result.stdout)["sql"]) + return outputs[0], outputs[1] + + +def sqlclean_formatted(*, sqlclean: str, statements: list[str]) -> list[str]: + """Each statement as `sqlclean format` lays it out for DuckDB at SQLQuack's indent and width.""" + with tempfile.TemporaryDirectory() as directory: + root: Path = Path(directory) + (root / "sqlclean.toml").write_text(SQLCLEAN_CONFIG) + paths: list[Path] = [root / f"{index:05}.sql" for index in range(len(statements))] + for path, statement in zip(paths, statements, strict=True): + path.write_text(statement) + subprocess.run([sqlclean, "format", directory], cwd=root, capture_output=True, check=False) + return [path.read_text().rstrip("\n") for path in paths] diff --git a/scripts/format_conformance/_helpers/layouts.py b/scripts/format_conformance/_helpers/layouts.py new file mode 100644 index 000000000..2f7a8c709 --- /dev/null +++ b/scripts/format_conformance/_helpers/layouts.py @@ -0,0 +1,49 @@ +"""Comparing the two formatters' layouts of each statement.""" + +from __future__ import annotations + +import difflib + + +def compared_layouts( + *, pairs: list[tuple[str, str, str]], theirs: list[str] +) -> tuple[dict[str, int], list[str]]: + """How many statements each formatter lays out alike, and a diff of each that differs.""" + counts: dict[str, int] = {"declined": 0, "exact": 0, "cased": 0} + diffs: list[str] = [] + for (case_id, plain, ours), clean in zip(pairs, theirs, strict=True): + outcome: str | None = layout_outcome(plain=plain, ours=ours, clean=clean) + if outcome is not None: + counts[outcome] += 1 + continue + lines: list[str] = list( + difflib.unified_diff( + clean.splitlines(), ours.splitlines(), "sqlclean", "sqlquack", lineterm="" + ) + ) + diffs.append(f"## {case_id}\n" + "\n".join(lines)) + counts["different"] = len(diffs) + return counts, diffs + + +def layout_outcome(*, plain: str, ours: str, clean: str) -> str | None: + """`declined` when SQLClean left it unformatted, `exact` or `cased` when alike, else None.""" + if clean == plain and ours != plain: + return "declined" + if ours == clean: + return "exact" + return "cased" if ours.lower() == clean.lower() else None + + +def summary_lines(counts: dict[str, int]) -> list[str]: + """The measurement's summary, as percentages of the statements both formatters laid out.""" + compared: int = max(sum(counts.values()) - counts["declined"], 1) + layout: int = counts["exact"] + counts["cased"] + return [ + f" left unformatted by SQLClean, not compared: {counts['declined']}", + f" same text: {counts['exact']} ({counts['exact'] / compared:.1%})", + f" same layout, keyword case only: {counts['cased']} ({counts['cased'] / compared:.1%})", + f" same layout in all: {layout} ({layout / compared:.1%})", + f" different layout: {counts['different']} " + f"({counts['different'] / compared:.1%})", + ] diff --git a/scripts/format_conformance/constants.py b/scripts/format_conformance/constants.py new file mode 100644 index 000000000..2d5f69138 --- /dev/null +++ b/scripts/format_conformance/constants.py @@ -0,0 +1,11 @@ +"""Constants of the formatter conformance measurement.""" + +from __future__ import annotations + +CASES_DIR: str = "cases" +SQLCLEAN_CONFIG: str = 'dialect = "duckdb"\n\n[format]\nindent = 4\nline_width = 100\n' +DIFFS_SHOWN: int = 5 +QUERY_KEY: str = "query" +COLUMNS_KEY: str = "columns" +INPUT_RELATION: str = "INPUT" +QUOTE: str = '"' diff --git a/scripts/format_conformance/main/__init__.py b/scripts/format_conformance/main/__init__.py new file mode 100644 index 000000000..b2aa50419 --- /dev/null +++ b/scripts/format_conformance/main/__init__.py @@ -0,0 +1 @@ +"""Entry points.""" diff --git a/scripts/format_conformance/main/compare.py b/scripts/format_conformance/main/compare.py new file mode 100644 index 000000000..38bd7c9b8 --- /dev/null +++ b/scripts/format_conformance/main/compare.py @@ -0,0 +1,41 @@ +"""Compare `sqlquack convert --pretty` with `sqlclean format` on the cases' converted SQL.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.format_conformance._helpers.cases import case_queries +from scripts.format_conformance._helpers.formatters import converted, sqlclean_formatted +from scripts.format_conformance._helpers.layouts import compared_layouts, summary_lines +from scripts.format_conformance.constants import CASES_DIR, DIFFS_SHOWN + + +def run_format_conformance(argv: list[str] | None = None) -> int: + """Print how often SQLQuack's layout matches SQLClean's; a measurement that exits 0.""" + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sqlquack", default="target/debug/sqlquack") + parser.add_argument("--sqlclean", default="sqlclean") + parser.add_argument("--cases", type=Path, default=Path(CASES_DIR)) + parser.add_argument("--diffs", type=Path, help="write every differing case's diff here") + arguments: argparse.Namespace = parser.parse_args(argv) + pairs: list[tuple[str, str, str]] = [] + for case_id, query, schema in case_queries(arguments.cases): + sql: tuple[str, str] | None = converted( + sqlquack=arguments.sqlquack, query=query, schema=schema + ) + if sql is not None: + pairs.append((case_id, *sql)) + theirs: list[str] = sqlclean_formatted( + sqlclean=arguments.sqlclean, statements=[plain for _, plain, _ in pairs] + ) + counts, diffs = compared_layouts(pairs=pairs, theirs=theirs) + print(f"format conformance: {len(pairs)} converted statements from {arguments.cases}") + for line in summary_lines(counts): + print(line) + if arguments.diffs: + arguments.diffs.write_text("\n\n".join(diffs) + "\n") + for diff in diffs[:DIFFS_SHOWN]: + print(f"\n{diff}", file=sys.stderr) + return 0 diff --git a/scripts/format_conformance/types.py b/scripts/format_conformance/types.py new file mode 100644 index 000000000..08fbdefca --- /dev/null +++ b/scripts/format_conformance/types.py @@ -0,0 +1,3 @@ +"""Types of the formatter conformance measurement.""" + +type Schema = dict[str, list[list[str]]] diff --git a/scripts/measure_format_conformance.py b/scripts/measure_format_conformance.py new file mode 100644 index 000000000..9ae35552b --- /dev/null +++ b/scripts/measure_format_conformance.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the formatter conformance measurement against SQLClean.""" + +from __future__ import annotations + +from scripts.format_conformance.main.compare import run_format_conformance + + +def main(argv: list[str] | None = None) -> int: + """Measure how often `sqlquack convert --pretty` lays SQL out as `sqlclean format` does.""" + return run_format_conformance(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index 12e056fbd..ae049506b 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -125,7 +125,8 @@ async function run() { spans.push({ kind: refused ? 'warn' : 'error', span: diagnostic.span, text: tone(rendered(diagnostic, chars)) }); let body = ''; if (translate.status !== 'skipped') { - body = (translate.statements ?? []).map((s) => terminal(escape(s.sql))).join(''); + const sql = (translate.statements ?? []).map((s) => s.pretty).join(';\n\n'); + if (sql) body = `
${terminal(escape(sql))}
`; body += (translate.diagnostics ?? []).map((d) => terminal(tone(rendered(d, chars)))).join(''); } const policy = translate.profile ? `policy: ${translate.profile}` : ''; @@ -204,6 +205,13 @@ const schedule = () => { document.addEventListener('click', (event) => { if (event.target.id === 'pg-run') execute(); + const copy = event.target.closest('[data-copy-sql]'); + if (copy) { + navigator.clipboard.writeText(copy.previousElementSibling.textContent).then(() => { + copy.textContent = 'Copied'; + setTimeout(() => (copy.textContent = 'Copy'), 1500); + }); + } }); sqlBox.addEventListener('input', () => { showCaption(null); diff --git a/website/public/styles.css b/website/public/styles.css index 57c328c95..3ed87ab92 100644 --- a/website/public/styles.css +++ b/website/public/styles.css @@ -1250,6 +1250,8 @@ footer a { .pg-stages { display: grid; + /* A zero minimum, so the unwrapped SQL scrolls inside its pane instead of widening the page. */ + grid-template-columns: minmax(0, 1fr); gap: 12px; align-content: start; } @@ -1359,6 +1361,26 @@ footer a { margin: 12px 16px; } +/* The translated SQL, laid out over lines, with its Copy button in the corner. */ +.pg-sql { + position: relative; +} + +/* The SQL is already laid out over lines, so it never wraps; a long line scrolls sideways. */ +.pg-body .pg-sql .terminal { + padding-right: 80px; + white-space: pre; + overflow-wrap: normal; + overflow-x: auto; +} + +.pg-body .pg-sql .copy { + position: absolute; + top: 10px; + right: 12px; + margin: 0; +} + .pg-rows { max-height: 280px; overflow: auto; diff --git a/website/src/content/docs/docs/cli/convert.mdx b/website/src/content/docs/docs/cli/convert.mdx index 1588d39a3..e6b119747 100644 --- a/website/src/content/docs/docs/cli/convert.mdx +++ b/website/src/content/docs/docs/cli/convert.mdx @@ -55,6 +55,7 @@ SQL that does not parse, or that Snowflake rejects, is refused with the same cod | `--session NAME=VALUE` | A Snowflake session parameter of your session. Repeatable. See [Sessions](/docs/policy/#sessions). | | `--overrides ` | User overrides: a `FUNCTION()` file, or a directory of them, each with `replaces` and `dialect duckdb`. | | `--verified-override NAME=SHA256` | An override verified for the body with that hash, as `convert --json` reports. | +| `--pretty` | Lay the DuckDB SQL out over lines, one clause per line. Only whitespace changes. | | `--format text\|json`, `--json` | Print the report as text or JSON. | `convert` also takes the policy flags: `--policy`, `--builtin-policy`, `--strict`, diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index cd7852c51..c88ddeea8 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -51,6 +51,13 @@ test('each stage reports pass, fail or skipped', () => { assert.deepEqual(stages('SELECT HASH(order_id) AS h FROM orders'), ['pass', 'pass', 'fail']); }); +test('translated SQL comes laid out over lines, with only whitespace changed', () => { + const report = JSON.parse(analyze('SELECT order_id, note FROM orders WHERE order_id > 1', SCHEMA)); + const [statement] = report.translate.statements; + assert.match(statement.pretty, /^SELECT\n {4}.+,\n {4}.+\nFROM .+\nWHERE /); + assert.equal(statement.pretty.replace(/\s+/g, ''), statement.sql.replace(/\s+/g, '')); +}); + test('the policy and session parameters choose the profile, or refuse, as on the command line', () => { const policy = readFileSync(new URL('../../sqlquack.toml', import.meta.url), 'utf8'); const translate = (settings) => { diff --git a/website/wasm/Cargo.lock b/website/wasm/Cargo.lock index d1890e908..38407f341 100644 --- a/website/wasm/Cargo.lock +++ b/website/wasm/Cargo.lock @@ -176,6 +176,13 @@ dependencies = [ "sqlquack-snowflake", ] +[[package]] +name = "sqlquack-format" +version = "0.2.0" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "sqlquack-policy" version = "0.2.0" @@ -246,6 +253,7 @@ dependencies = [ "serde_json", "sqlquack-core", "sqlquack-duckdb", + "sqlquack-format", "sqlquack-policy", "sqlquack-rules", "sqlquack-semantics", diff --git a/website/wasm/Cargo.toml b/website/wasm/Cargo.toml index 2ea3af4fd..cad999c83 100644 --- a/website/wasm/Cargo.toml +++ b/website/wasm/Cargo.toml @@ -18,6 +18,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sqlquack-core = { path = "../../crates/sqlquack-core" } sqlquack-duckdb = { path = "../../crates/sqlquack-duckdb", default-features = false } +sqlquack-format = { path = "../../crates/sqlquack-format" } sqlquack-policy = { path = "../../crates/sqlquack-policy" } sqlquack-rules = { path = "../../crates/sqlquack-rules", default-features = false } sqlquack-semantics = { path = "../../crates/sqlquack-semantics" } diff --git a/website/wasm/src/lib.rs b/website/wasm/src/lib.rs index ddc57a502..d63de4c38 100644 --- a/website/wasm/src/lib.rs +++ b/website/wasm/src/lib.rs @@ -23,6 +23,7 @@ use sqlquack_core::refusals::models::{Refusal, RefusalCategory}; use sqlquack_duckdb::blocks::main::column_type::column_type; use sqlquack_duckdb::blocks::main::quoted_identifier::quoted_identifier; use sqlquack_duckdb::checking::constants::SEMI_STRUCTURED_HOLDING_TYPE; +use sqlquack_format::pretty_sql; use sqlquack_policy::policy::models::Policy; use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; use sqlquack_rules::conversion::main::convert_statement::convert_statement; @@ -306,6 +307,7 @@ fn report(sql: &str, schema: &str, settings: &str) -> Value { let elapsed = Duration::from_secs_f64((now_ms() - started).max(0.0) / 1000.0); match done.and_then(|done| within_budget(done, (sql, elapsed), &options)) { Ok(done) => converted.push(json!({ + "pretty": pretty_sql(&done.sql, Dialect::DuckDB), "sql": done.sql, "columns": done.columns.iter().map(|column| json!({ "name": column.name, From 54ceaaa4acdc7b604cecb560af23f6754b1e2c0f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:29:48 +0000 Subject: [PATCH 02/12] test(corpus): record everyday Snowflake queries' stage outcomes Add corpora/everyday: 514 neutral, synthetic queries over a shop schema (literals, SELECT *, joins, CTEs, aggregates, windows and QUALIFY, dates, IFF/CASE/COALESCE, strings, semi-structured paths and FLATTEN, dbt-style models, DDL and DML, typos and invalid SQL). Each records what SQLQuack does with it under the built-in default policy: its parse or check diagnostic, its first refusal, or translated. Queries whose Snowflake validity is not yet recorded are marked. A test runs each through check_source and convert_report, the paths the command line and Python use, and fails naming every query whose outcome changed. --- Cargo.lock | 1 + corpora/everyday/queries.toml | 2750 +++++++++++++++++ corpora/everyday/schema.json | 196 ++ crates/sqlquack/Cargo.toml | 1 + crates/sqlquack/src/analysis/tests.rs | 2 + crates/sqlquack/src/analysis/tests/helpers.rs | 71 +- .../analysis/tests/test_everyday_corpus.rs | 19 + .../sqlquack/src/analysis/tests/test_types.rs | 18 + 8 files changed, 3057 insertions(+), 1 deletion(-) create mode 100644 corpora/everyday/queries.toml create mode 100644 corpora/everyday/schema.json create mode 100644 crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs diff --git a/Cargo.lock b/Cargo.lock index 720a0a79f..bb96d19d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1361,6 +1361,7 @@ dependencies = [ "sqlquack-snowflake", "sqlquack-syntax", "tempfile", + "toml", "walkdir", ] diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml new file mode 100644 index 000000000..9acf20f61 --- /dev/null +++ b/corpora/everyday/queries.toml @@ -0,0 +1,2750 @@ +# Everyday Snowflake queries, as a demo visitor or a dbt project writes them, over schema.json. +# Each records its outcome from SQLQuack under the built-in default policy, which the playground +# uses: "translated", "parse ", "check " or "refused ". The test +# fails when an outcome changes; update `expect` when a change is intended. `uncertain` marks a +# query whose Snowflake validity is not yet recorded. + +[[query]] +category = "literals" +sql = '''SELECT 1''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "literals" +sql = '''SELECT 1 AS one''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 'hello'''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "literals" +sql = '''SELECT 'hello' AS greeting''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 1, 2, 3''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "literals" +sql = '''SELECT 1 + 1''' +expect = "refused SQ0003 node:add" + +[[query]] +category = "literals" +sql = '''SELECT 1 + 1 AS two''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 10 / 4 AS ratio''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 10 % 3 AS remainder''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT -5 AS negative''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 3.14 AS pi''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 1e3 AS thousand''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT TRUE AS yes, FALSE AS no''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT NULL AS nothing''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT NULL''' +expect = "refused SQ0003 node:null" + +[[query]] +category = "literals" +sql = '''SELECT 'it''s' AS escaped''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 'a' || 'b' AS joined''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT CURRENT_DATE''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "literals" +sql = '''SELECT CURRENT_DATE AS today''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "literals" +sql = '''SELECT CURRENT_TIMESTAMP AS now''' +expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" + +[[query]] +category = "literals" +sql = '''SELECT CURRENT_DATE()''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "literals" +sql = '''SELECT DATE '2024-01-15' AS d''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT TIMESTAMP '2024-01-15 10:30:00' AS ts''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT '2024-01-15'::DATE AS d''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT '42'::INT AS n''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT CAST('42' AS NUMBER) AS n''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT CAST(3.7 AS INTEGER) AS n''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 1 AS a UNION ALL SELECT 2''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS t(id, label)''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT column1, column2 FROM VALUES (1, 'a'), (2, 'b')''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 2 * (3 + 4) AS n''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT ROUND(2.567, 2) AS r''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT ABS(-3) AS a''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT UPPER('abc') AS u''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT LENGTH('hello') AS n''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 1 WHERE 1 = 1''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "literals" +sql = '''select 1 as lower_case''' +expect = "translated" + +[[query]] +category = "literals" +sql = '''SELECT 1 AS "Quoted Name"''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM ORDERS''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders LIMIT 10''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders o''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT o.* FROM orders o''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders WHERE status = 'shipped'''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders ORDER BY ordered_at DESC LIMIT 5''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * EXCLUDE (discount) FROM orders''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * RENAME (total AS order_total) FROM orders''' +expect = "refused SQ0002 node:star" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM customers WHERE country = 'DE'''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT DISTINCT * FROM products''' +expect = "translated" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders SAMPLE (10 ROWS)''' +expect = "refused SQ0002 node:table" + +[[query]] +category = "select_star" +sql = '''SELECT * FROM orders TOP 5''' +expect = "parse SQ0001" +uncertain = true + +[[query]] +category = "select_star" +sql = '''SELECT TOP 5 * FROM orders''' +expect = "refused SQ0002 node:select" + +[[query]] +category = "select_star" +sql = '''SELECT o.*, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, customer_id, status, total FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT o.order_id, o.total FROM orders AS o''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT orders.order_id FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT ORDER_ID, TOTAL FROM ORDERS''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT "ORDER_ID" FROM "ORDERS"''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id AS id, total AS amount FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id id, total amount FROM orders''' +expect = "translated" +uncertain = true + +[[query]] +category = "columns" +sql = '''SELECT total * 1.2 AS gross FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT total - discount AS net FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT total - COALESCE(discount, 0) AS net FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT total / 2 AS half FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT total, total * 0.19 AS vat FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT first_name || ' ' || last_name AS full_name FROM customers''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT CONCAT(first_name, ' ', last_name) AS full_name FROM customers''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT DISTINCT status FROM orders''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT DISTINCT country FROM customers ORDER BY country''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE total > 100''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE total BETWEEN 10 AND 100''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE status IN ('shipped', 'delivered')''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE status NOT IN ('cancelled')''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE shipped_at IS NULL''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE shipped_at IS NOT NULL AND total > 0''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE status = 'shipped' OR status = 'delivered'''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE NOT (status = 'cancelled')''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT email FROM customers WHERE email LIKE '%@example.com'''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT email FROM customers WHERE email ILIKE '%@EXAMPLE.COM'''' +expect = "refused SQ0008 node:i_like" + +[[query]] +category = "columns" +sql = '''SELECT email FROM customers WHERE email NOT LIKE '%test%'''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT name FROM products WHERE name RLIKE '^[A-Z].*'''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT customer_id FROM customers WHERE is_active''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT customer_id FROM customers WHERE NOT is_active''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT customer_id FROM customers WHERE is_active = TRUE''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders ORDER BY total DESC''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders ORDER BY 2 DESC''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders ORDER BY total DESC NULLS LAST''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders ORDER BY total LIMIT 10 OFFSET 20''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders LIMIT 1''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders FETCH FIRST 10 ROWS ONLY''' +expect = "refused SQ0002 node:select" +uncertain = true + +[[query]] +category = "columns" +sql = '''SELECT order_id, total FROM orders WHERE total = (SELECT MAX(total) FROM orders)''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id FROM orders WHERE customer_id IN (SELECT customer_id FROM customers WHERE country = 'US')''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT customer_id FROM customers c WHERE EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id)''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT customer_id FROM customers c WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id)''' +expect = "translated" + +[[query]] +category = "columns" +sql = '''SELECT order_id, (SELECT COUNT(*) FROM order_items i WHERE i.order_id = o.order_id) AS lines FROM orders o''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(*) FROM orders''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(*) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(1) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(order_id) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(DISTINCT customer_id) AS customers FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(total) AS revenue FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT AVG(total) AS average FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT MIN(total) AS smallest, MAX(total) AS largest FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT MIN(ordered_at) AS first_order, MAX(ordered_at) AS last_order FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT status, COUNT(*) FROM orders GROUP BY status''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY status''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY 1''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY n DESC''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id HAVING SUM(total) > 1000''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, COUNT(*) AS orders FROM orders GROUP BY customer_id HAVING COUNT(*) > 1''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, SUM(total) AS revenue, COUNT(*) AS orders, AVG(total) AS aov FROM orders GROUP BY customer_id''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT channel, status, COUNT(*) AS n FROM orders GROUP BY channel, status''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT country, COUNT(*) AS customers FROM customers GROUP BY country ORDER BY customers DESC LIMIT 10''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT category, AVG(price) AS avg_price FROM products GROUP BY category''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT category, ROUND(AVG(price), 2) AS avg_price FROM products GROUP BY category''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT category, MEDIAN(price) AS median_price FROM products GROUP BY category''' +expect = "refused SQ0008 fn:MEDIAN" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(quantity * unit_price) AS revenue FROM order_items''' +expect = "refused SQ0003 node:mul" + +[[query]] +category = "aggregates" +sql = '''SELECT order_id, SUM(quantity) AS units FROM order_items GROUP BY order_id''' +expect = "refused SQ0003 fn:SUM" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT_IF(status = 'cancelled') AS cancelled FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(IFF(status = 'cancelled', 1, 0)) AS cancelled FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders''' +expect = "refused SQ0002 fn:COUNT" +uncertain = true + +[[query]] +category = "aggregates" +sql = '''SELECT ANY_VALUE(status) AS some_status FROM orders''' +expect = "refused SQ0008 fn:ANY_VALUE" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, LISTAGG(status, ', ') AS statuses FROM orders GROUP BY customer_id''' +expect = "refused SQ0002 fn:LISTAGG" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, LISTAGG(DISTINCT status, ',') WITHIN GROUP (ORDER BY status) AS statuses FROM orders GROUP BY customer_id''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, ARRAY_AGG(order_id) AS order_ids FROM orders GROUP BY customer_id''' +expect = "refused SQ0008 fn:ARRAY_AGG" + +[[query]] +category = "aggregates" +sql = '''SELECT customer_id, ARRAY_AGG(order_id) WITHIN GROUP (ORDER BY ordered_at) AS order_ids FROM orders GROUP BY customer_id''' +expect = "refused SQ0008 fn:ARRAY_AGG" + +[[query]] +category = "aggregates" +sql = '''SELECT STDDEV(total) AS sd, VARIANCE(total) AS var FROM orders''' +expect = "refused SQ0002 fn:STDDEV" + +[[query]] +category = "aggregates" +sql = '''SELECT APPROX_COUNT_DISTINCT(customer_id) AS approx_customers FROM orders''' +expect = "refused SQ0002 fn:APPROX_COUNT_DISTINCT" + +[[query]] +category = "aggregates" +sql = '''SELECT BOOLOR_AGG(is_active) AS any_active FROM customers''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT MAX_BY(order_id, total) AS biggest_order FROM orders''' +expect = "refused SQ0008 fn:MAX_BY" + +[[query]] +category = "aggregates" +sql = '''SELECT MODE(status) AS common_status FROM orders''' +expect = "refused SQ0008 fn:MODE" + +[[query]] +category = "aggregates" +sql = '''SELECT PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY total) AS p90 FROM orders''' +expect = "refused SQ0008 fn:PERCENTILE_CONT" + +[[query]] +category = "aggregates" +sql = '''SELECT channel, SUM(total) AS revenue FROM orders GROUP BY ROLLUP (channel)''' +expect = "refused SQ0002 fn:ROLLUP" + +[[query]] +category = "aggregates" +sql = '''SELECT channel, status, COUNT(*) AS n FROM orders GROUP BY GROUPING SETS ((channel), (status))''' +expect = "refused SQ0002 fn:GROUPING SETS" + +[[query]] +category = "aggregates" +sql = '''SELECT channel, COUNT(*) AS n FROM orders GROUP BY ALL''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month, SUM(total) AS revenue FROM orders GROUP BY 1 ORDER BY 1''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month, COUNT(DISTINCT customer_id) AS active_customers FROM orders GROUP BY month''' +expect = "refused SQ0003 node:group_by" + +[[query]] +category = "aggregates" +sql = '''SELECT YEAR(ordered_at) AS yr, SUM(total) AS revenue FROM orders GROUP BY yr ORDER BY yr''' +expect = "refused SQ0003 node:group_by" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(total) FROM orders''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT MAX(total) FROM orders''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT AVG(total) FROM orders''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT status, SUM(total) FROM orders GROUP BY status''' +expect = "refused SQ0003 node:aggregate_function" + +[[query]] +category = "aggregates" +sql = '''SELECT COUNT(*) AS n, SUM(total) / COUNT(*) AS aov FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT SUM(total) / NULLIF(COUNT(*), 0) AS aov FROM orders''' +expect = "translated" + +[[query]] +category = "aggregates" +sql = '''SELECT DIV0(SUM(total), COUNT(*)) AS aov FROM orders''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o INNER JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o LEFT JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o LEFT OUTER JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o RIGHT JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o FULL OUTER JOIN customers c ON o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT c.customer_id, o.order_id FROM customers c CROSS JOIN orders o''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o, customers c WHERE o.customer_id = c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c USING (customer_id)''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT order_id, email FROM orders JOIN customers USING (customer_id)''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o NATURAL JOIN customers c''' +expect = "refused SQ0002 join:natural" + +[[query]] +category = "joins" +sql = '''SELECT orders.order_id, customers.email FROM orders JOIN customers ON orders.customer_id = customers.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, i.line_number, p.name FROM orders o JOIN order_items i ON o.order_id = i.order_id JOIN products p ON i.product_id = p.product_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT c.email, o.order_id, i.quantity, p.name, pay.method FROM customers c JOIN orders o ON o.customer_id = c.customer_id JOIN order_items i ON i.order_id = o.order_id JOIN products p ON p.product_id = i.product_id LEFT JOIN payments pay ON pay.order_id = o.order_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT c.customer_id, COUNT(o.order_id) AS orders FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT c.customer_id, COALESCE(SUM(o.total), 0) AS revenue FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT p.category, SUM(i.quantity * i.unit_price) AS revenue FROM order_items i JOIN products p ON p.product_id = i.product_id GROUP BY p.category ORDER BY revenue DESC''' +expect = "refused SQ0003 node:mul" + +[[query]] +category = "joins" +sql = '''SELECT c.customer_id FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id WHERE o.order_id IS NULL''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id FROM orders o JOIN payments p ON p.order_id = o.order_id AND p.amount = o.total''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, p.amount FROM orders o LEFT JOIN payments p ON p.order_id = o.order_id AND p.method = 'card'''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT a.order_id, b.order_id AS next_order FROM orders a JOIN orders b ON a.customer_id = b.customer_id AND b.ordered_at > a.ordered_at''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.country FROM orders o JOIN customers c ON o.customer_id = c.customer_id WHERE c.country IN ('US', 'CA') AND o.total > 50''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT t.ticket_id, c.email FROM tickets t JOIN customers c ON c.customer_id = t.customer_id WHERE t.closed_at IS NULL''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id ORDER BY o.ordered_at DESC LIMIT 20''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, x.total_paid FROM orders o JOIN (SELECT order_id, SUM(amount) AS total_paid FROM payments GROUP BY order_id) x ON x.order_id = o.order_id''' +expect = "translated" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id, x.total_paid FROM orders o LEFT JOIN LATERAL (SELECT SUM(amount) AS total_paid FROM payments p WHERE p.order_id = o.order_id) x''' +expect = "refused SQ0002 node:subquery" + +[[query]] +category = "joins" +sql = '''SELECT o.order_id FROM orders o ASOF JOIN payments p MATCH_CONDITION (o.ordered_at >= p.paid_at) ON o.order_id = p.order_id''' +expect = "translated" +uncertain = true + +[[query]] +category = "ctes" +sql = '''WITH t AS (SELECT 1 AS x) SELECT x FROM t''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH t AS (SELECT 1 AS x) SELECT * FROM t''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH recent AS (SELECT * FROM orders WHERE ordered_at >= '2024-01-01') SELECT COUNT(*) AS n FROM recent''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH shipped AS (SELECT order_id, customer_id, total FROM orders WHERE status = 'shipped') SELECT customer_id, SUM(total) AS revenue FROM shipped GROUP BY customer_id''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH revenue AS (SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id), ranked AS (SELECT customer_id, revenue, RANK() OVER (ORDER BY revenue DESC) AS rnk FROM revenue) SELECT * FROM ranked WHERE rnk <= 10''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH a AS (SELECT customer_id FROM orders), b AS (SELECT customer_id FROM customers) SELECT COUNT(*) AS n FROM a JOIN b USING (customer_id)''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH totals AS (SELECT order_id, SUM(quantity * unit_price) AS items_total FROM order_items GROUP BY order_id) SELECT o.order_id, o.total, t.items_total, o.total - t.items_total AS diff FROM orders o JOIN totals t ON t.order_id = o.order_id''' +expect = "refused SQ0003 node:mul" + +[[query]] +category = "ctes" +sql = '''WITH firsts AS (SELECT customer_id, MIN(ordered_at) AS first_order_at FROM orders GROUP BY customer_id) SELECT DATE_TRUNC('month', first_order_at) AS cohort, COUNT(*) AS customers FROM firsts GROUP BY 1''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH t (a, b) AS (SELECT 1, 2) SELECT a + b AS s FROM t''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "ctes" +sql = '''WITH RECURSIVE n AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM n WHERE i < 10) SELECT i FROM n''' +expect = "refused SQ0002 node:with" + +[[query]] +category = "ctes" +sql = '''WITH x AS (SELECT order_id, total FROM orders) SELECT order_id FROM x WHERE total > (SELECT AVG(total) FROM x)''' +expect = "translated" + +[[query]] +category = "ctes" +sql = '''WITH paid AS (SELECT order_id, SUM(amount) AS paid FROM payments GROUP BY order_id) SELECT o.order_id, o.total, COALESCE(p.paid, 0) AS paid, o.total - COALESCE(p.paid, 0) AS outstanding FROM orders o LEFT JOIN paid p ON p.order_id = o.order_id WHERE o.total > COALESCE(p.paid, 0)''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, ROW_NUMBER() OVER (ORDER BY ordered_at) AS rn FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, customer_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS order_number FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, RANK() OVER (ORDER BY total DESC) AS rnk FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, DENSE_RANK() OVER (ORDER BY total DESC) AS rnk FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, total, SUM(total) OVER (PARTITION BY customer_id) AS customer_total FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, total, SUM(total) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS running_total FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, total, SUM(total) OVER (ORDER BY ordered_at ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, AVG(total) OVER (ORDER BY ordered_at ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS moving_avg FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, LAG(total) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS previous_total FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, LEAD(ordered_at) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS next_order_at FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, LAG(total, 1, 0) OVER (ORDER BY ordered_at) AS prev FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, FIRST_VALUE(order_id) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS first_order FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, LAST_VALUE(order_id) OVER (PARTITION BY customer_id ORDER BY ordered_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS last_order FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, NTILE(4) OVER (ORDER BY total) AS quartile FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, total / SUM(total) OVER () AS share FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, RATIO_TO_REPORT(total) OVER () AS share FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, COUNT(*) OVER (PARTITION BY customer_id) AS customer_orders FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, PERCENT_RANK() OVER (ORDER BY total) AS pr FROM orders''' +expect = "refused SQ0002 fn:PERCENT_RANK" + +[[query]] +category = "windows" +sql = '''SELECT order_id, CUME_DIST() OVER (ORDER BY total) AS cd FROM orders''' +expect = "refused SQ0002 fn:CUME_DIST" + +[[query]] +category = "windows" +sql = '''SELECT * FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) = 1''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, customer_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) = 1''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, customer_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) AS rn FROM orders QUALIFY rn = 1''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT customer_id, order_id FROM orders QUALIFY RANK() OVER (PARTITION BY customer_id ORDER BY total DESC) <= 3''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id FROM (SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) AS rn FROM orders) WHERE rn = 1''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, DATEDIFF('day', LAG(ordered_at) OVER (PARTITION BY customer_id ORDER BY ordered_at), ordered_at) AS days_since_previous FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at, order_id) AS seq FROM orders''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT DATE_TRUNC('day', ordered_at) AS day, SUM(SUM(total)) OVER (ORDER BY DATE_TRUNC('day', ordered_at)) AS cumulative FROM orders GROUP BY 1''' +expect = "translated" + +[[query]] +category = "windows" +sql = '''SELECT order_id, CONDITIONAL_TRUE_EVENT(status = 'cancelled') OVER (ORDER BY ordered_at) AS grp FROM orders''' +expect = "refused SQ0002 fn:CONDITIONAL_TRUE_EVENT" + +[[query]] +category = "dates" +sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATE_TRUNC('week', ordered_at) AS week FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATE_TRUNC('day', ordered_at) AS day FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATE_TRUNC(month, ordered_at) AS month FROM orders''' +expect = "translated" +uncertain = true + +[[query]] +category = "dates" +sql = '''SELECT ordered_at::DATE AS order_date FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TO_DATE(ordered_at) AS order_date FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT CAST(ordered_at AS DATE) AS order_date FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATE(ordered_at) AS order_date FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATEADD(day, 7, ordered_at) AS due_at FROM orders''' +expect = "refused SQ0008 fn:DATEADD" + +[[query]] +category = "dates" +sql = '''SELECT DATEADD('day', 7, ordered_at) AS due_at FROM orders''' +expect = "refused SQ0008 fn:DATEADD" + +[[query]] +category = "dates" +sql = '''SELECT DATEADD(month, 1, signup_date) AS next_month FROM customers''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT ordered_at + INTERVAL '1 day' AS tomorrow FROM orders''' +expect = "refused SQ0008 node:add" + +[[query]] +category = "dates" +sql = '''SELECT signup_date + 30 AS trial_end FROM customers''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATEDIFF(day, ordered_at, shipped_at) AS days_to_ship FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATEDIFF('hour', ordered_at, shipped_at) AS hours_to_ship FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TIMESTAMPDIFF(minute, ordered_at, shipped_at) AS minutes FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT YEAR(ordered_at) AS y, MONTH(ordered_at) AS m, DAY(ordered_at) AS d FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT EXTRACT(year FROM ordered_at) AS y FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATE_PART('dow', ordered_at) AS dow FROM orders''' +expect = "translated" +uncertain = true + +[[query]] +category = "dates" +sql = '''SELECT DAYOFWEEK(ordered_at) AS dow FROM orders''' +expect = "refused SQ0002 fn:DAYOFWEEK" + +[[query]] +category = "dates" +sql = '''SELECT DAYNAME(ordered_at) AS day_name FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT MONTHNAME(ordered_at) AS month_name FROM orders''' +expect = "refused SQ0002 fn:MONTHNAME" + +[[query]] +category = "dates" +sql = '''SELECT QUARTER(ordered_at) AS q FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT WEEKOFYEAR(ordered_at) AS w FROM orders''' +expect = "refused SQ0002 fn:WEEKOFYEAR" + +[[query]] +category = "dates" +sql = '''SELECT LAST_DAY(signup_date) AS month_end FROM customers''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TO_CHAR(ordered_at, 'YYYY-MM-DD') AS day FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TO_VARCHAR(ordered_at, 'YYYY-MM') AS month FROM orders''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TO_TIMESTAMP('2024-01-15 10:00:00') AS ts''' +expect = "refused SQ0008 fn:TO_TIMESTAMP" + +[[query]] +category = "dates" +sql = '''SELECT TO_DATE('2024-01-15') AS d''' +expect = "refused SQ0008 fn:TO_DATE" + +[[query]] +category = "dates" +sql = '''SELECT TO_DATE('15/01/2024', 'DD/MM/YYYY') AS d''' +expect = "refused SQ0008 fn:TO_DATE" + +[[query]] +category = "dates" +sql = '''SELECT TRY_TO_DATE('not a date') AS d''' +expect = "refused SQ0008 fn:TRY_TO_DATE" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE ordered_at >= '2024-01-01'''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_DATE)''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_TIMESTAMP())''' +expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE ordered_at BETWEEN '2024-01-01' AND '2024-01-31'''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE ordered_at::DATE = CURRENT_DATE - 1''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "dates" +sql = '''SELECT order_id FROM orders WHERE DATE_TRUNC('month', ordered_at) = DATE_TRUNC('month', CURRENT_DATE)''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "dates" +sql = '''SELECT customer_id FROM customers WHERE signup_date > '2023-06-30'::DATE''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT DATEDIFF(year, signup_date, CURRENT_DATE) AS tenure_years FROM customers''' +expect = "refused SQ0008 fn:CURRENT_DATE" + +[[query]] +category = "dates" +sql = '''SELECT CONVERT_TIMEZONE('UTC', 'America/New_York', ordered_at) AS local_time FROM orders''' +expect = "refused SQ0008 fn:CONVERT_TIMEZONE" + +[[query]] +category = "dates" +sql = '''SELECT DATE_FROM_PARTS(2024, 1, 15) AS d''' +expect = "translated" + +[[query]] +category = "dates" +sql = '''SELECT TIME_SLICE(ordered_at, 15, 'MINUTE') AS slot FROM orders''' +expect = "refused SQ0002 fn:TIME_SLICE" + +[[query]] +category = "dates" +sql = '''SELECT SYSDATE() AS now''' +expect = "refused SQ0002 fn:SYSDATE" + +[[query]] +category = "dates" +sql = '''SELECT GETDATE() AS now''' +expect = "refused SQ0002 fn:GETDATE" + +[[query]] +category = "dates" +sql = '''SELECT ADD_MONTHS(signup_date, 3) AS plus3 FROM customers''' +expect = "refused SQ0002 fn:ADD_MONTHS" + +[[query]] +category = "dates" +sql = '''SELECT MONTHS_BETWEEN(CURRENT_DATE, signup_date) AS months FROM customers''' +expect = "refused SQ0002 fn:MONTHS_BETWEEN" + +[[query]] +category = "dates" +sql = '''SELECT order_id, opened_at FROM tickets t JOIN orders o ON DATE_TRUNC('day', t.opened_at) = DATE_TRUNC('day', o.ordered_at)''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, IFF(total > 100, 'large', 'small') AS size FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, CASE WHEN total > 100 THEN 'large' ELSE 'small' END AS size FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, CASE WHEN total > 500 THEN 'xl' WHEN total > 100 THEN 'l' WHEN total > 20 THEN 'm' ELSE 's' END AS size FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, CASE status WHEN 'shipped' THEN 1 WHEN 'delivered' THEN 2 ELSE 0 END AS stage FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, CASE WHEN discount IS NULL THEN 0 ELSE discount END AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, COALESCE(discount, 0) AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, NVL(discount, 0) AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, IFNULL(discount, 0) AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, ZEROIFNULL(discount) AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, NULLIF(discount, 0) AS discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, NVL2(shipped_at, 'shipped', 'pending') AS state FROM orders''' +expect = "refused SQ0002 fn:NVL2" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, COALESCE(shipped_at, ordered_at) AS last_event_at FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, GREATEST(total, discount) AS g, LEAST(total, discount) AS l FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, DECODE(status, 'shipped', 1, 'delivered', 2, 0) AS stage FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, IFF(shipped_at IS NULL, NULL, DATEDIFF(day, ordered_at, shipped_at)) AS days FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT customer_id, IFF(is_active, 'active', 'inactive') AS state FROM customers''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT customer_id, CASE WHEN country IN ('DE', 'FR', 'IT') THEN 'EU' WHEN country = 'US' THEN 'NA' ELSE 'Other' END AS region FROM customers''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, total > 100 AS is_large FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, EQUAL_NULL(discount, NULL) AS no_discount FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, discount IS DISTINCT FROM 0 AS has_discount FROM orders''' +expect = "translated" +uncertain = true + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, IFF(total > 100, total * 0.9, total) AS charged FROM orders''' +expect = "translated" + +[[query]] +category = "conditionals" +sql = '''SELECT order_id, CASE WHEN total >= 100 AND channel = 'web' THEN 'promo' END AS promo FROM orders''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT UPPER(email) AS e FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LOWER(email) AS e FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT TRIM(first_name) AS f FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LTRIM(first_name) AS f, RTRIM(last_name) AS l FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LENGTH(email) AS n FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LEN(email) AS n FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT SUBSTR(email, 1, 5) AS prefix FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT SUBSTRING(email, 1, 5) AS prefix FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LEFT(sku, 3) AS family, RIGHT(sku, 4) AS suffix FROM products''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT SPLIT_PART(email, '@', 2) AS domain FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT SPLIT_PART(email, '@', 1) AS local_part FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT POSITION('@' IN email) AS at_pos FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT CHARINDEX('@', email) AS at_pos FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT REPLACE(sku, '-', '') AS clean_sku FROM products''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT REGEXP_REPLACE(sku, '[^A-Z0-9]', '') AS clean_sku FROM products''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT REGEXP_SUBSTR(email, '@(.*)$', 1, 1, 'e') AS domain FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT REGEXP_LIKE(email, '.*@example\\.com') AS is_example FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT INITCAP(first_name) AS f FROM customers''' +expect = "refused SQ0008 fn:INITCAP" + +[[query]] +category = "strings" +sql = '''SELECT CONCAT_WS(' ', first_name, last_name) AS full_name FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT LPAD(CAST(order_id AS VARCHAR), 8, '0') AS padded FROM orders''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT order_id::VARCHAR AS id_text FROM orders''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT TO_VARCHAR(order_id) AS id_text FROM orders''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT TO_VARCHAR(total, '999,999.00') AS formatted FROM orders''' +expect = "refused SQ0003 fn:TO_VARCHAR" + +[[query]] +category = "strings" +sql = '''SELECT TO_NUMBER('123.45', 10, 2) AS n''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT TRY_TO_NUMBER('abc') AS n''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT TRY_CAST('abc' AS INTEGER) AS n''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT STARTSWITH(email, 'admin') AS is_admin FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT ENDSWITH(email, '.com') AS dot_com FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT CONTAINS(email, 'test') AS is_test FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT REVERSE(sku) AS r FROM products''' +expect = "refused SQ0002 fn:REVERSE" + +[[query]] +category = "strings" +sql = '''SELECT REPEAT('-', 10) AS line''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT MD5(email) AS hashed FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT SHA2(email, 256) AS hashed FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT HASH(email) AS h FROM customers''' +expect = "refused SQ0008 fn:HASH" + +[[query]] +category = "strings" +sql = '''SELECT SPLIT(email, '@') AS parts FROM customers''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT STRTOK(email, '@', 1) AS local_part FROM customers''' +expect = "refused SQ0002 fn:STRTOK" + +[[query]] +category = "strings" +sql = '''SELECT ASCII(first_name) AS code FROM customers''' +expect = "refused SQ0002 fn:ASCII" + +[[query]] +category = "strings" +sql = '''SELECT UUID_STRING() AS id''' +expect = "refused SQ0002 fn:UUID_STRING" + +[[query]] +category = "strings" +sql = '''SELECT email FROM customers WHERE LOWER(email) = 'a@example.com'''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT email FROM customers WHERE TRIM(LOWER(email)) LIKE '%example%'''' +expect = "translated" + +[[query]] +category = "strings" +sql = '''SELECT COLLATE(email, 'en-ci') AS e FROM customers''' +expect = "refused SQ0002 fn:COLLATE" +uncertain = true + +[[query]] +category = "strings" +sql = '''SELECT EDITDISTANCE(first_name, last_name) AS d FROM customers''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT ROUND(total) AS r FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT ROUND(total, 1) AS r FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT FLOOR(total) AS f, CEIL(total) AS c FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT TRUNC(total) AS t FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT TRUNCATE(total, 1) AS t FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT MOD(order_id, 10) AS bucket FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT order_id % 10 AS bucket FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT POWER(total, 2) AS sq FROM orders''' +expect = "refused SQ0003 fn:POWER" + +[[query]] +category = "numeric" +sql = '''SELECT SQRT(total) AS root FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT LN(total) AS l FROM orders WHERE total > 0''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT LOG(10, total) AS l FROM orders WHERE total > 0''' +expect = "refused SQ0002 fn:LOG" + +[[query]] +category = "numeric" +sql = '''SELECT EXP(1) AS e''' +expect = "refused SQ0008 fn:EXP" + +[[query]] +category = "numeric" +sql = '''SELECT SIGN(total - 100) AS s FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT total::FLOAT AS f FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT total::NUMBER(10,0) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT CAST(total AS DECIMAL(10, 1)) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT CAST(total AS INT) AS n FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT weight_kg * 1000 AS grams FROM products''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT ROUND(weight_kg, 2) AS w FROM products''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT price * 1.0 / 3 AS third FROM products''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT quantity * unit_price AS line_total FROM order_items''' +expect = "refused SQ0003 node:mul" + +[[query]] +category = "numeric" +sql = '''SELECT SUM(quantity) * 1.0 / COUNT(DISTINCT order_id) AS units_per_order FROM order_items''' +expect = "refused SQ0003 fn:SUM" + +[[query]] +category = "numeric" +sql = '''SELECT UNIFORM(1, 10, RANDOM()) AS r''' +expect = "refused SQ0002 fn:UNIFORM" + +[[query]] +category = "numeric" +sql = '''SELECT RANDOM() AS r''' +expect = "refused SQ0002 fn:RANDOM" + +[[query]] +category = "numeric" +sql = '''SELECT order_id FROM orders WHERE total::INT = 100''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT BITAND(order_id, 1) AS odd FROM orders''' +expect = "translated" + +[[query]] +category = "numeric" +sql = '''SELECT WIDTH_BUCKET(total, 0, 1000, 10) AS bucket FROM orders''' +expect = "refused SQ0002 fn:WIDTH_BUCKET" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page::STRING AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page::VARCHAR AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:device.os::STRING AS os FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:items[0].sku::STRING AS first_sku FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload['page']::STRING AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:"page"::STRING AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:amount::NUMBER(10,2) AS amount FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:amount::FLOAT AS amount FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT event_id FROM events WHERE payload:page::STRING = '/checkout'''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page::STRING AS page, COUNT(*) AS views FROM events WHERE event_type = 'page_view' GROUP BY 1 ORDER BY 2 DESC''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT GET(payload, 'page') AS page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT GET_PATH(payload, 'device.os') AS os FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT TRY_PARSE_JSON('{"a": 1}') AS j''' +expect = "refused SQ0008 fn:TRY_PARSE_JSON" + +[[query]] +category = "semi_structured" +sql = '''SELECT PARSE_JSON('{"a": 1}'):a::INT AS a''' +expect = "refused SQ0008 fn:PARSE_JSON" + +[[query]] +category = "semi_structured" +sql = '''SELECT OBJECT_CONSTRUCT('id', order_id, 'total', total) AS o FROM orders''' +expect = "refused SQ0008 fn:OBJECT_CONSTRUCT" + +[[query]] +category = "semi_structured" +sql = '''SELECT ARRAY_CONSTRUCT(1, 2, 3) AS arr''' +expect = "refused SQ0008 fn:ARRAY_CONSTRUCT" + +[[query]] +category = "semi_structured" +sql = '''SELECT ARRAY_SIZE(tags) AS n FROM tickets''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT ARRAY_CONTAINS('urgent'::VARIANT, tags) AS urgent FROM tickets''' +expect = "refused SQ0008 node:column" +uncertain = true + +[[query]] +category = "semi_structured" +sql = '''SELECT tags[0]::STRING AS first_tag FROM tickets''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT ticket_id, f.value::STRING AS tag FROM tickets, LATERAL FLATTEN(input => tags) f''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT ticket_id, f.value::STRING AS tag FROM tickets t, TABLE(FLATTEN(t.tags)) f''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT e.event_id, i.value:sku::STRING AS sku, i.value:qty::INT AS qty FROM events e, LATERAL FLATTEN(input => e.payload:items) i''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT e.event_id, i.index, i.value FROM events e, LATERAL FLATTEN(input => e.payload:items) i''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT event_id, OBJECT_KEYS(payload) AS keys FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT TYPEOF(payload) AS t FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT IS_NULL_VALUE(payload:page) AS null_page FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload:page IS NULL AS missing FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT event_id FROM events WHERE payload:amount::NUMBER > 100''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT TO_JSON(payload) AS txt FROM events''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT payload FROM events LIMIT 10''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT ARRAY_TO_STRING(tags, ',') AS tag_list FROM tickets''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT customer_id, ARRAY_AGG(DISTINCT event_type) AS types FROM events GROUP BY customer_id''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "semi_structured" +sql = '''SELECT customer_id, OBJECT_AGG(event_type, payload) AS by_type FROM events GROUP BY customer_id''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "dbt_models" +sql = ''' +with source as ( + select * from orders +), +renamed as ( + select + order_id, + customer_id, + status as order_status, + total as order_total, + ordered_at + from source +) +select * from renamed''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +with orders as ( + select * from orders +), +payments as ( + select order_id, sum(amount) as amount from payments group by 1 +), +final as ( + select + orders.order_id, + orders.customer_id, + orders.ordered_at, + coalesce(payments.amount, 0) as amount + from orders + left join payments on orders.order_id = payments.order_id +) +select * from final''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +with customer_orders as ( + select + customer_id, + min(ordered_at) as first_order_at, + max(ordered_at) as most_recent_order_at, + count(order_id) as number_of_orders, + sum(total) as lifetime_value + from orders + group by 1 +) +select + customers.customer_id, + customers.first_name, + customers.last_name, + customer_orders.first_order_at, + customer_orders.most_recent_order_at, + coalesce(customer_orders.number_of_orders, 0) as number_of_orders, + coalesce(customer_orders.lifetime_value, 0) as lifetime_value +from customers +left join customer_orders using (customer_id)''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +select + date_trunc('day', ordered_at) as order_date, + channel, + count(*) as orders, + sum(total) as revenue, + sum(total) / nullif(count(*), 0) as average_order_value +from orders +where status != 'cancelled' +group by 1, 2''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +with events_ranked as ( + select + *, + row_number() over (partition by event_id order by occurred_at desc) as rn + from events +) +select * exclude (rn) from events_ranked where rn = 1''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "dbt_models" +sql = ''' +select + order_id, + customer_id, + case + when status in ('returned', 'refunded') then 'returned' + when shipped_at is not null then 'fulfilled' + else 'open' + end as fulfillment_state, + datediff('day', ordered_at, coalesce(shipped_at, current_timestamp())) as age_days +from orders''' +expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" + +[[query]] +category = "dbt_models" +sql = ''' +with daily as ( + select ordered_at::date as day, sum(total) as revenue + from orders + group by 1 +) +select + day, + revenue, + avg(revenue) over (order by day rows between 6 preceding and current row) as revenue_7d_avg +from daily''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +select + p.product_id, + p.name, + p.category, + sum(i.quantity) as units_sold, + sum(i.quantity * i.unit_price) as gross_revenue, + count(distinct i.order_id) as orders +from products p +left join order_items i on i.product_id = p.product_id +group by 1, 2, 3''' +expect = "refused SQ0003 fn:SUM" + +[[query]] +category = "dbt_models" +sql = ''' +with ticket_durations as ( + select + ticket_id, + priority, + datediff('minute', opened_at, closed_at) as minutes_open + from tickets + where closed_at is not null +) +select priority, avg(minutes_open) as avg_minutes, max(minutes_open) as max_minutes +from ticket_durations +group by priority''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "dbt_models" +sql = ''' +select + customer_id, + count_if(event_type = 'page_view') as page_views, + count_if(event_type = 'add_to_cart') as add_to_carts, + count_if(event_type = 'purchase') as purchases +from events +group by customer_id''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "dbt_models" +sql = ''' +with monthly as ( + select customer_id, date_trunc('month', ordered_at) as month, sum(total) as revenue + from orders + group by 1, 2 +) +select + customer_id, + month, + revenue, + lag(revenue) over (partition by customer_id order by month) as previous_revenue, + revenue - lag(revenue) over (partition by customer_id order by month) as change +from monthly''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +select + {{ dbt_utils.generate_surrogate_key(['order_id', 'line_number']) }} as order_item_key, + order_id +from {{ ref('stg_order_items') }}''' +expect = "parse SQ0001" + +[[query]] +category = "dbt_models" +sql = ''' +select + md5(cast(order_id as varchar) || '-' || cast(line_number as varchar)) as order_item_key, + order_id, + line_number +from order_items''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +select * +from orders +where ordered_at > (select coalesce(max(ordered_at), '1900-01-01') from orders)''' +expect = "refused SQ0003 fn:COALESCE" + +[[query]] +category = "dbt_models" +sql = ''' +with spine as ( + select dateadd(day, seq4(), '2024-01-01'::date) as day + from table(generator(rowcount => 31)) +) +select spine.day, count(o.order_id) as orders +from spine +left join orders o on o.ordered_at::date = spine.day +group by 1''' +expect = "translated" + +[[query]] +category = "dbt_models" +sql = ''' +select + customer_id, + email, + country, + signup_date, + is_active, + created_at +from customers +where email is not null + and country is not null''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id FROM orders UNION SELECT customer_id FROM customers''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id FROM orders UNION ALL SELECT customer_id FROM tickets''' +expect = "refused SQ0008 node:column" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id FROM customers EXCEPT SELECT customer_id FROM orders''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id FROM customers MINUS SELECT customer_id FROM orders''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id FROM customers INTERSECT SELECT customer_id FROM orders''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT 'orders' AS source, COUNT(*) AS n FROM orders UNION ALL SELECT 'customers', COUNT(*) FROM customers''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT order_id, total FROM orders WHERE total > 100 UNION ALL SELECT order_id, total FROM orders WHERE total < 10 ORDER BY total''' +expect = "translated" + +[[query]] +category = "set_operations" +sql = '''SELECT customer_id, email FROM customers UNION ALL BY NAME SELECT email, customer_id FROM customers''' +expect = "translated" +uncertain = true + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders WHERE order_id = 1;''' +expect = "translated" + +[[query]] +category = "misc" +sql = '''SELECT 1; SELECT 2''' +expect = "refused SQ0003 node:literal" + +[[query]] +category = "misc" +sql = ''' +SELECT order_id -- the id +FROM orders''' +expect = "translated" + +[[query]] +category = "misc" +sql = '''/* comment */ SELECT order_id FROM orders''' +expect = "translated" + +[[query]] +category = "misc" +sql = '''SELECT COUNT(*) AS n FROM orders WHERE status ILIKE ANY ('ship%', 'deliv%')''' +expect = "refused SQ0008 node:i_like" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders WHERE status LIKE ANY ('ship%', 'deliv%')''' +expect = "refused SQ0003 node:like" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders ORDER BY RANDOM() LIMIT 1''' +expect = "refused SQ0002 fn:RANDOM" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders WHERE (customer_id, status) IN (SELECT customer_id, 'shipped' FROM customers)''' +expect = "refused SQ0002 node:tuple" +uncertain = true + +[[query]] +category = "misc" +sql = '''SELECT o.order_id FROM orders o WHERE o.total > ALL (SELECT amount FROM payments)''' +expect = "refused SQ0002 node:all" + +[[query]] +category = "misc" +sql = '''SELECT IDENTIFIER('orders') AS x''' +expect = "refused SQ0002 fn:IDENTIFIER" + +[[query]] +category = "misc" +sql = '''SELECT * FROM IDENTIFIER('orders')''' +expect = "refused SQ0002 node:table_function" + +[[query]] +category = "misc" +sql = '''SELECT CURRENT_USER() AS u''' +expect = "refused SQ0002 fn:CURRENT_USER" + +[[query]] +category = "misc" +sql = '''SELECT CURRENT_WAREHOUSE() AS w''' +expect = "refused SQ0002 fn:CURRENT_WAREHOUSE" + +[[query]] +category = "misc" +sql = '''SELECT $1 FROM orders''' +expect = "refused SQ0003 node:parameter" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders AT(OFFSET => -60)''' +expect = "refused SQ0002 node:table" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders WHERE customer_id = :customer_id''' +expect = "refused SQ0002 node:parameter" + +[[query]] +category = "misc" +sql = '''SELECT order_id FROM orders WHERE customer_id = ?''' +expect = "refused SQ0002 node:placeholder" + +[[query]] +category = "misc" +sql = '''SELECT SEQ4() AS n FROM TABLE(GENERATOR(ROWCOUNT => 5))''' +expect = "translated" + +[[query]] +category = "misc" +sql = '''SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p''' +expect = "refused SQ0003 node:pivot" +uncertain = true + +[[query]] +category = "misc" +sql = '''SELECT * FROM (SELECT channel, status, total FROM orders) PIVOT (SUM(total) FOR status IN ('shipped', 'pending'))''' +expect = "refused SQ0003 node:pivot" + +[[query]] +category = "misc" +sql = '''SELECT order_id, HASH_AGG(*) AS h FROM orders GROUP BY order_id''' +expect = "refused SQ0002 fn:HASH_AGG" +uncertain = true + +[[query]] +category = "misc" +sql = '''SELECT CURRENT_ACCOUNT() AS a''' +expect = "refused SQ0002 fn:CURRENT_ACCOUNT" + +[[query]] +category = "ddl_dml" +sql = '''CREATE TABLE big_orders AS SELECT * FROM orders WHERE total > 1000''' +expect = "refused SQ0002 node:create_table" + +[[query]] +category = "ddl_dml" +sql = '''CREATE OR REPLACE TABLE big_orders AS SELECT order_id, total FROM orders WHERE total > 1000''' +expect = "refused SQ0002 node:create_table" + +[[query]] +category = "ddl_dml" +sql = '''CREATE VIEW v_orders AS SELECT order_id, total FROM orders''' +expect = "refused SQ0002 node:create_view" + +[[query]] +category = "ddl_dml" +sql = '''CREATE TEMPORARY TABLE t AS SELECT 1 AS x''' +expect = "refused SQ0002 node:create_table" + +[[query]] +category = "ddl_dml" +sql = '''INSERT INTO orders (order_id, customer_id, status, total) VALUES (1, 1, 'new', 10.00)''' +expect = "refused SQ0002 node:insert" + +[[query]] +category = "ddl_dml" +sql = '''INSERT INTO orders SELECT * FROM orders WHERE order_id = 1''' +expect = "refused SQ0002 node:insert" + +[[query]] +category = "ddl_dml" +sql = '''UPDATE orders SET status = 'cancelled' WHERE order_id = 1''' +expect = "refused SQ0002 node:update" + +[[query]] +category = "ddl_dml" +sql = '''DELETE FROM orders WHERE status = 'cancelled'''' +expect = "refused SQ0002 node:delete" + +[[query]] +category = "ddl_dml" +sql = '''MERGE INTO orders t USING orders s ON t.order_id = s.order_id WHEN MATCHED THEN UPDATE SET t.status = s.status WHEN NOT MATCHED THEN INSERT (order_id, status) VALUES (s.order_id, s.status)''' +expect = "refused SQ0002 node:merge" + +[[query]] +category = "ddl_dml" +sql = '''TRUNCATE TABLE orders''' +expect = "refused SQ0002 node:truncate" + +[[query]] +category = "ddl_dml" +sql = '''DROP TABLE IF EXISTS big_orders''' +expect = "refused SQ0002 node:drop_table" + +[[query]] +category = "ddl_dml" +sql = '''CREATE TABLE t (id INT, name VARCHAR)''' +expect = "refused SQ0002 node:create_table" + +[[query]] +category = "ddl_dml" +sql = '''ALTER TABLE orders ADD COLUMN note VARCHAR''' +expect = "refused SQ0002 node:alter_table" + +[[query]] +category = "ddl_dml" +sql = '''SHOW TABLES''' +expect = "refused SQ0002 node:show" + +[[query]] +category = "ddl_dml" +sql = '''DESCRIBE TABLE orders''' +expect = "refused SQ0002 node:describe" + +[[query]] +category = "ddl_dml" +sql = '''USE WAREHOUSE compute_wh''' +expect = "refused SQ0002 node:use" + +[[query]] +category = "invalid_syntax" +sql = '''SELEC 1''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT * FORM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT * FROM''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id total FROM orders WHERE''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id, FROM orders''' +expect = "translated" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id,, total FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE total >''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE status = 'shipped''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE (total > 10''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE total > 10)''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders GROUP status''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders ORDER total''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders LIMIT''' +expect = "refused SQ0003 node:limit" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT COUNT(* FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT SUM(total)) FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders o JOIN customers c''' +expect = "refused SQ0002 join:inner" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders o JOIN customers c ON''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE status == 'shipped'''' +expect = "translated" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE status = "shipped"''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT CASE WHEN total > 1 THEN 'a' FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT CASE total > 1 THEN 'a' END FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE total BETWEEN 1''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE status IN 'shipped'''' +expect = "refused SQ0002 node:in" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE status IN ()''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''WITH t AS SELECT 1 SELECT * FROM t''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''WITH t AS (SELECT 1) SELECT * FROM''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders UNION''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT * FROM orders WHERE''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders HAVING''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT DISTINCT FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT 1 +''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT 'unterminated''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders;;''' +expect = "translated" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id AS FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE total >> 10''' +expect = "refused SQ0002 node:bitwise_right_shift" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders o LEFT orders c ON 1 = 1''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT row_number() OVER (PARTITION customer_id) FROM orders''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders QUALIFY''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT IFF(total > 10, 'a') AS x FROM orders''' +expect = "refused SQ0002 fn:IFF" + +[[query]] +category = "invalid_syntax" +sql = '''SELECT order_id FROM orders WHERE order_id IN (1, 2,)''' +expect = "parse SQ0001" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT discount_code FROM orders''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM order''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orderz''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT o.order_id FROM orders''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT x.order_id FROM orders o''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT custmer_id FROM customers''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id, status FROM orders GROUP BY status''' +expect = "refused SQ0002 node:statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT status, total FROM orders GROUP BY status''' +expect = "refused SQ0002 node:statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT customer_id FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders WHERE SUM(total) > 10''' +expect = "refused SQ0003 fn:SUM" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT COUNT(*) AS n FROM orders WHERE n > 1''' +expect = "refused SQ0002 node:statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT SUM(SUM(total)) AS s FROM orders''' +expect = "refused SQ0003 fn:SUM" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders ORDER BY 5''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders GROUP BY 3''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders WHERE ROW_NUMBER() OVER (ORDER BY total) = 1''' +expect = "refused SQ0003 node:window_function" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id, total + 'abc' AS x FROM orders''' +expect = "refused SQ0003 node:add" +uncertain = true + +[[query]] +category = "invalid_semantics" +sql = '''SELECT DATEADD(day, 'x', ordered_at) AS x FROM orders''' +expect = "refused SQ0003 fn:DATEADD" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders WHERE ordered_at > 5''' +expect = "refused SQ0003 node:gt" +uncertain = true + +[[query]] +category = "invalid_semantics" +sql = '''SELECT LENGTH(order_id, 2) AS x FROM orders''' +expect = "refused SQ0002 fn:LENGTH" +uncertain = true + +[[query]] +category = "invalid_semantics" +sql = '''SELECT NOSUCHFUNCTION(order_id) AS x FROM orders''' +expect = "refused SQ0002 fn:NOSUCHFUNCTION" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT 1 QUALIFY TRUE''' +expect = "check SQ0203" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id QUALIFY revenue > 100''' +expect = "check SQ0203" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT payload:page FROM orders''' +expect = "refused SQ0004 statement" + +[[query]] +category = "invalid_semantics" +sql = '''SELECT order_id FROM orders o, orders o''' +expect = "refused SQ0004 statement" +uncertain = true diff --git a/corpora/everyday/schema.json b/corpora/everyday/schema.json new file mode 100644 index 000000000..ecaf57019 --- /dev/null +++ b/corpora/everyday/schema.json @@ -0,0 +1,196 @@ +{ + "CUSTOMERS": [ + [ + "CUSTOMER_ID", + "NUMBER(38,0)" + ], + [ + "FIRST_NAME", + "VARCHAR" + ], + [ + "LAST_NAME", + "VARCHAR" + ], + [ + "EMAIL", + "VARCHAR" + ], + [ + "COUNTRY", + "VARCHAR" + ], + [ + "SIGNUP_DATE", + "DATE" + ], + [ + "IS_ACTIVE", + "BOOLEAN" + ], + [ + "CREATED_AT", + "TIMESTAMP_NTZ" + ] + ], + "ORDERS": [ + [ + "ORDER_ID", + "NUMBER(38,0)" + ], + [ + "CUSTOMER_ID", + "NUMBER(38,0)" + ], + [ + "STATUS", + "VARCHAR" + ], + [ + "TOTAL", + "NUMBER(12,2)" + ], + [ + "DISCOUNT", + "NUMBER(12,2)" + ], + [ + "ORDERED_AT", + "TIMESTAMP_NTZ" + ], + [ + "SHIPPED_AT", + "TIMESTAMP_NTZ" + ], + [ + "CHANNEL", + "VARCHAR" + ] + ], + "ORDER_ITEMS": [ + [ + "ORDER_ID", + "NUMBER(38,0)" + ], + [ + "LINE_NUMBER", + "NUMBER(38,0)" + ], + [ + "PRODUCT_ID", + "NUMBER(38,0)" + ], + [ + "QUANTITY", + "NUMBER(38,0)" + ], + [ + "UNIT_PRICE", + "NUMBER(10,2)" + ] + ], + "PRODUCTS": [ + [ + "PRODUCT_ID", + "NUMBER(38,0)" + ], + [ + "SKU", + "VARCHAR" + ], + [ + "NAME", + "VARCHAR" + ], + [ + "CATEGORY", + "VARCHAR" + ], + [ + "PRICE", + "NUMBER(10,2)" + ], + [ + "WEIGHT_KG", + "FLOAT" + ], + [ + "LAUNCHED_ON", + "DATE" + ] + ], + "PAYMENTS": [ + [ + "PAYMENT_ID", + "NUMBER(38,0)" + ], + [ + "ORDER_ID", + "NUMBER(38,0)" + ], + [ + "METHOD", + "VARCHAR" + ], + [ + "AMOUNT", + "NUMBER(12,2)" + ], + [ + "PAID_AT", + "TIMESTAMP_NTZ" + ] + ], + "EVENTS": [ + [ + "EVENT_ID", + "VARCHAR" + ], + [ + "CUSTOMER_ID", + "NUMBER(38,0)" + ], + [ + "EVENT_TYPE", + "VARCHAR" + ], + [ + "PAYLOAD", + "VARIANT" + ], + [ + "OCCURRED_AT", + "TIMESTAMP_NTZ" + ] + ], + "TICKETS": [ + [ + "TICKET_ID", + "NUMBER(38,0)" + ], + [ + "CUSTOMER_ID", + "NUMBER(38,0)" + ], + [ + "PRIORITY", + "VARCHAR" + ], + [ + "SUBJECT", + "VARCHAR" + ], + [ + "OPENED_AT", + "TIMESTAMP_NTZ" + ], + [ + "CLOSED_AT", + "TIMESTAMP_NTZ" + ], + [ + "TAGS", + "ARRAY" + ] + ] +} diff --git a/crates/sqlquack/Cargo.toml b/crates/sqlquack/Cargo.toml index 5648c9fd6..5b4f39310 100644 --- a/crates/sqlquack/Cargo.toml +++ b/crates/sqlquack/Cargo.toml @@ -38,6 +38,7 @@ walkdir.workspace = true arrow.workspace = true duckdb.workspace = true tempfile.workspace = true +toml.workspace = true [lints] workspace = true diff --git a/crates/sqlquack/src/analysis/tests.rs b/crates/sqlquack/src/analysis/tests.rs index e9be0a968..4f49765c4 100644 --- a/crates/sqlquack/src/analysis/tests.rs +++ b/crates/sqlquack/src/analysis/tests.rs @@ -24,6 +24,8 @@ mod test_divergence_modes; mod test_emitted_growth; #[path = "tests/test_equivalence_cases.rs"] mod test_equivalence_cases; +#[path = "tests/test_everyday_corpus.rs"] +mod test_everyday_corpus; #[path = "tests/test_float_text.rs"] mod test_float_text; #[path = "tests/test_guarded_conditions.rs"] diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index 7d6e59467..77adbe9c7 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -24,13 +24,16 @@ use sqlquack_syntax::{Dialect, TokenKind, tokenize}; use crate::analysis::main::analyze::analyze; use crate::analysis::main::analyze_function::analyze_function; use crate::analysis::main::check_equivalence_cases::check_equivalence_cases; +use crate::analysis::main::check_source::check_source; use crate::analysis::main::collect_all_refusals::collect_all_refusals; use crate::analysis::main::confirm_assumption::confirm_assumption; use crate::analysis::main::refused_construct::refused_construct; use crate::analysis::models::PipelineConverter; use crate::analysis::tests::test_types::{ - AnalysisSummary, CollectedSummary, DivergenceModeOutcome, + AnalysisSummary, CollectedSummary, DivergenceModeOutcome, EverydayCorpus, }; +use crate::cli::main::convert_report::convert_report; +use crate::cli::models::{ConvertArguments, OutputArguments, OutputFormat, PolicyArguments}; use sqlquack_rules::session::main::confirm_utc_session::confirm_utc_session; use sqlquack_rules::session::main::conversion_options::conversion_options; @@ -779,3 +782,69 @@ const PARSED_TREE: &str = r#"{"error":false"#; /// The start of DuckDB's source-offset fields, `query_location` and `query_location_length`. const LOCATION_FIELD: &str = r#""query_location"#; + +/// Each everyday corpus query whose outcome differs from its recorded `expect`, with both. +pub(crate) fn everyday_outcome_mismatches() -> Vec { + let root = repository_root().join(EVERYDAY_CORPUS); + let text = std::fs::read_to_string(root.join("queries.toml")).expect("the corpus reads"); + let corpus: EverydayCorpus = toml::from_str(&text).expect("the corpus parses"); + assert!(corpus.query.len() > 400, "{} queries", corpus.query.len()); + let arguments = ConvertArguments { + path: None, + schema: Some(root.join("schema.json")), + overrides: None, + verified_overrides: Vec::new(), + session: Vec::new(), + pretty: false, + output: OutputArguments { + format: OutputFormat::Json, + json: false, + }, + conversion: PolicyArguments { + policy: POLICY_FILE_NAME.into(), + builtin_policy: true, + strict: false, + utc_session: false, + confirm_declared_offsets: false, + python_runtime: false, + loud_divergences_only: false, + }, + }; + corpus + .query + .iter() + .map(|query| (query, everyday_outcome(&query.sql, &arguments))) + .filter(|(query, outcome)| *outcome != query.expect) + .map(|(query, outcome)| format!("{}: expected {}, got {outcome}", query.sql, query.expect)) + .collect() +} + +/// What SQLQuack does with `sql`: its first check diagnostic, first refusal, or `translated`. +fn everyday_outcome(sql: &str, arguments: &ConvertArguments) -> String { + let diagnostics = check_source(sql).expect("the query holds a statement"); + diagnostics.first().map_or_else( + || { + let report = convert_report(arguments, sql).expect("the query converts or refuses"); + report.refusals.first().map_or_else( + || TRANSLATED.to_owned(), + |refusal| { + let code = refusal.diagnostic.as_ref().map_or("", |found| found.code); + format!("refused {code} {}", refusal.construct) + }, + ) + }, + |found| { + let stage = (found.code == PARSE_FAILURE_CODE).then_some("parse"); + format!("{} {}", stage.unwrap_or("check"), found.code) + }, + ) +} + +/// The corpus directory, under the repository root. +const EVERYDAY_CORPUS: &str = "corpora/everyday"; + +/// The outcome of a query that converts. +const TRANSLATED: &str = "translated"; + +/// The diagnostic code of SQL that does not parse. +const PARSE_FAILURE_CODE: &str = "SQ0001"; diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs new file mode 100644 index 000000000..ee68a37eb --- /dev/null +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -0,0 +1,19 @@ +use crate::analysis::tests::helpers::everyday_outcome_mismatches; +use crate::analysis::tests::test_types::EverydayCorpusTestCase; + +#[test] +fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_its_outcome() { + let test_cases = [EverydayCorpusTestCase { + description: "every corpora/everyday query parses, checks, converts or refuses as recorded", + expected_mismatches: Vec::new(), + }]; + + for test_case in test_cases { + assert_eq!( + everyday_outcome_mismatches(), + test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/analysis/tests/test_types.rs b/crates/sqlquack/src/analysis/tests/test_types.rs index d97677dc6..f0268816f 100644 --- a/crates/sqlquack/src/analysis/tests/test_types.rs +++ b/crates/sqlquack/src/analysis/tests/test_types.rs @@ -210,3 +210,21 @@ pub(crate) struct OverrideCallsTestCase { pub(crate) emitted: &'static str, pub(crate) expected_refused: bool, } + +pub(crate) struct EverydayCorpusTestCase { + pub(crate) description: &'static str, + pub(crate) expected_mismatches: Vec, +} + +/// The everyday corpus file. +#[derive(serde::Deserialize)] +pub(crate) struct EverydayCorpus { + pub(crate) query: Vec, +} + +/// One everyday query and the outcome recorded for it. +#[derive(serde::Deserialize)] +pub(crate) struct EverydayQuery { + pub(crate) sql: String, + pub(crate) expect: String, +} From facdcefd41c3445f8022405cf92363c3b1fdd9c7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:33:31 +0000 Subject: [PATCH 03/12] fix(playground): do not translate SQL the name checks reject The browser engine has no DuckDB bind, so SQL with a column missing from GROUP BY, or an aggregate in WHERE, showed stage 2 failing and stage 3 translating, where the command line refuses it. Translation is now skipped when the checks report an error; warnings do not stop it. --- website/tests/playground.test.mjs | 17 +++++++++-------- website/wasm/src/lib.rs | 9 ++++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index c88ddeea8..2d40f575b 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -17,16 +17,14 @@ test('spans after non-ASCII text underline the characters the diagnostic names', // 'é' is one UTF-16 unit, '😀' two and 'ß' one: three characters but four units before `nope`. const sql = "SELECT 'é😀ß' AS note, nope FROM orders"; const report = JSON.parse(analyze(sql, SCHEMA)); - const [refusal] = report.translate.diagnostics; - assert.equal(refusal.code, 'SQ0004'); + const [finding] = report.checks.findings; + assert.equal(finding.code, 'E201'); const chars = Array.from(sql); - assert.equal(chars.slice(refusal.span.start, refusal.span.end).join(''), 'nope'); + assert.equal(chars.slice(finding.span.start, finding.span.end).join(''), 'nope'); // Slicing the string by UTF-16 units would land one unit early, on ` nop`. - assert.notEqual(sql.slice(refusal.span.start, refusal.span.end), 'nope'); - const { html } = underlined(chars, [{ kind: 'error', span: refusal.span }]); + assert.notEqual(sql.slice(finding.span.start, finding.span.end), 'nope'); + const { html } = underlined(chars, [{ kind: 'error', span: finding.span }]); assert.deepEqual(marked(html), ['nope']); - const [finding] = report.checks.findings; - assert.equal(chars.slice(finding.span.start, finding.span.end).join(''), 'nope'); }); test('a parse failure on a later line renders at its line and column', () => { @@ -49,6 +47,9 @@ test('each stage reports pass, fail or skipped', () => { assert.deepEqual(stages('SELECT order_id FROM orders WHERE'), ['fail', 'skipped', 'skipped']); assert.deepEqual(stages('SELECT 1 QUALIFY TRUE'), ['pass', 'fail', 'skipped']); assert.deepEqual(stages('SELECT HASH(order_id) AS h FROM orders'), ['pass', 'pass', 'fail']); + // SQL Snowflake rejects is not translated, though no rule refuses it: DuckDB's bind would. + assert.deepEqual(stages('SELECT order_id, note FROM orders GROUP BY note'), ['pass', 'fail', 'skipped']); + assert.deepEqual(stages('SELECT COUNT(*) AS n FROM orders WHERE n > 1'), ['pass', 'fail', 'skipped']); }); test('translated SQL comes laid out over lines, with only whitespace changed', () => { @@ -96,7 +97,7 @@ test('every example stops at the stage it is meant to, and the default translate decimal: ['pass', 'pass', 'pass'], syntax: ['fail', 'skipped', 'skipped'], rejected: ['pass', 'fail', 'skipped'], - unknown: ['pass', 'fail', 'fail'], + unknown: ['pass', 'fail', 'skipped'], refused: ['pass', 'pass', 'fail'], }; assert.deepEqual(Object.keys(EXAMPLES).sort(), Object.keys(expected).sort()); diff --git a/website/wasm/src/lib.rs b/website/wasm/src/lib.rs index d63de4c38..f55df3877 100644 --- a/website/wasm/src/lib.rs +++ b/website/wasm/src/lib.rs @@ -267,7 +267,14 @@ fn report(sql: &str, schema: &str, settings: &str) -> Value { "names_checked", json!(!catalog.columns.is_empty() && statements.len() == 1), ); - if !rejections.is_empty() { + // SQL Snowflake rejects is not translated, as the native check would refuse it at DuckDB's + // bind, which the browser does not run. Warnings (W codes) do not stop translation. + let erred = findings.iter().any(|finding| { + finding["code"] + .as_str() + .is_some_and(|code| code.starts_with('E')) + }); + if !rejections.is_empty() || erred { return json!({ "parse": parse, "checks": checks, "translate": Stage::skipped() }); } From 5f750ee2a2ab93dc012555034cede3b491f3a3cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:41:02 +0000 Subject: [PATCH 04/12] feat(playground): default to the repository policy under a UTC session The playground now starts on the repository's sqlquack.toml with a UTC session, and stage 3's header names the policy in force. A one-click switch turns to strict, the built-in policy that refuses every known divergence, so refusing rather than guessing stays easy to show; ?policy=strict opens the page that way. The everyday corpus records each query's outcome under both policies (expect and expect_repository), and its test checks both and the translation rate of each: 285 of 434 valid queries strict, 354 of 434 under the repository policy. --- corpora/everyday/queries.toml | 523 +++++++++++++++++- crates/sqlquack/src/analysis/tests/helpers.rs | 90 ++- .../analysis/tests/test_everyday_corpus.rs | 11 +- .../sqlquack/src/analysis/tests/test_types.rs | 3 + website/public/playground/index.html | 4 +- website/public/playground/playground.js | 36 +- website/public/styles.css | 18 +- website/tests/playground.test.mjs | 20 +- 8 files changed, 672 insertions(+), 33 deletions(-) diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 9acf20f61..0e17c5e5f 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -1,1958 +1,2347 @@ # Everyday Snowflake queries, as a demo visitor or a dbt project writes them, over schema.json. -# Each records its outcome from SQLQuack under the built-in default policy, which the playground -# uses: "translated", "parse ", "check " or "refused ". The test -# fails when an outcome changes; update `expect` when a change is intended. `uncertain` marks a -# query whose Snowflake validity is not yet recorded. +# `expect` is its outcome from SQLQuack under the built-in default policy, which the playground +# offers as "strict": "translated", "parse ", "check " or "refused ". +# `expect_repository` is the outcome under the repository's sqlquack.toml with a UTC session, the +# playground's default. The test fails when an outcome changes; update the expectation when a +# change is intended. `uncertain` marks a query whose Snowflake validity is not yet recorded. [[query]] category = "literals" sql = '''SELECT 1''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "literals" sql = '''SELECT 1 AS one''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 'hello'''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "literals" sql = '''SELECT 'hello' AS greeting''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 1, 2, 3''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "literals" sql = '''SELECT 1 + 1''' expect = "refused SQ0003 node:add" +expect_repository = "refused SQ0003 node:add" [[query]] category = "literals" sql = '''SELECT 1 + 1 AS two''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 10 / 4 AS ratio''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 10 % 3 AS remainder''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT -5 AS negative''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 3.14 AS pi''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 1e3 AS thousand''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT TRUE AS yes, FALSE AS no''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT NULL AS nothing''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT NULL''' expect = "refused SQ0003 node:null" +expect_repository = "refused SQ0003 node:null" [[query]] category = "literals" sql = '''SELECT 'it''s' AS escaped''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 'a' || 'b' AS joined''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT CURRENT_DATE''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "refused SQ0003 node:function" [[query]] category = "literals" sql = '''SELECT CURRENT_DATE AS today''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT CURRENT_TIMESTAMP AS now''' expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT CURRENT_DATE()''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "refused SQ0003 node:function" [[query]] category = "literals" sql = '''SELECT DATE '2024-01-15' AS d''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT TIMESTAMP '2024-01-15 10:30:00' AS ts''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT '2024-01-15'::DATE AS d''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT '42'::INT AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT CAST('42' AS NUMBER) AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT CAST(3.7 AS INTEGER) AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 1 AS a UNION ALL SELECT 2''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT * FROM (VALUES (1, 'a'), (2, 'b')) AS t(id, label)''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT column1, column2 FROM VALUES (1, 'a'), (2, 'b')''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 2 * (3 + 4) AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT ROUND(2.567, 2) AS r''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT ABS(-3) AS a''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT UPPER('abc') AS u''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT LENGTH('hello') AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 1 WHERE 1 = 1''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "literals" sql = '''select 1 as lower_case''' expect = "translated" +expect_repository = "translated" [[query]] category = "literals" sql = '''SELECT 1 AS "Quoted Name"''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM ORDERS''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders LIMIT 10''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders o''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT o.* FROM orders o''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders WHERE status = 'shipped'''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders ORDER BY ordered_at DESC LIMIT 5''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * EXCLUDE (discount) FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * RENAME (total AS order_total) FROM orders''' expect = "refused SQ0002 node:star" +expect_repository = "refused SQ0002 node:star" [[query]] category = "select_star" sql = '''SELECT * FROM customers WHERE country = 'DE'''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT DISTINCT * FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "select_star" sql = '''SELECT * FROM orders SAMPLE (10 ROWS)''' expect = "refused SQ0002 node:table" +expect_repository = "refused SQ0002 node:table" [[query]] category = "select_star" sql = '''SELECT * FROM orders TOP 5''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" uncertain = true [[query]] category = "select_star" sql = '''SELECT TOP 5 * FROM orders''' expect = "refused SQ0002 node:select" +expect_repository = "refused SQ0002 node:select" [[query]] category = "select_star" sql = '''SELECT o.*, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, customer_id, status, total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT o.order_id, o.total FROM orders AS o''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT orders.order_id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT ORDER_ID, TOTAL FROM ORDERS''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT "ORDER_ID" FROM "ORDERS"''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id AS id, total AS amount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id id, total amount FROM orders''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "columns" sql = '''SELECT total * 1.2 AS gross FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT total - discount AS net FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT total - COALESCE(discount, 0) AS net FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT total / 2 AS half FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT total, total * 0.19 AS vat FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT first_name || ' ' || last_name AS full_name FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT CONCAT(first_name, ' ', last_name) AS full_name FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT DISTINCT status FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT DISTINCT country FROM customers ORDER BY country''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE total > 100''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE total BETWEEN 10 AND 100''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE status IN ('shipped', 'delivered')''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE status NOT IN ('cancelled')''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE shipped_at IS NULL''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE shipped_at IS NOT NULL AND total > 0''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE status = 'shipped' OR status = 'delivered'''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE NOT (status = 'cancelled')''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT email FROM customers WHERE email LIKE '%@example.com'''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT email FROM customers WHERE email ILIKE '%@EXAMPLE.COM'''' expect = "refused SQ0008 node:i_like" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT email FROM customers WHERE email NOT LIKE '%test%'''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT name FROM products WHERE name RLIKE '^[A-Z].*'''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT customer_id FROM customers WHERE is_active''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT customer_id FROM customers WHERE NOT is_active''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT customer_id FROM customers WHERE is_active = TRUE''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders ORDER BY total DESC''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders ORDER BY 2 DESC''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders ORDER BY total DESC NULLS LAST''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders ORDER BY total LIMIT 10 OFFSET 20''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders LIMIT 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders FETCH FIRST 10 ROWS ONLY''' expect = "refused SQ0002 node:select" +expect_repository = "refused SQ0002 node:select" uncertain = true [[query]] category = "columns" sql = '''SELECT order_id, total FROM orders WHERE total = (SELECT MAX(total) FROM orders)''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id FROM orders WHERE customer_id IN (SELECT customer_id FROM customers WHERE country = 'US')''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT customer_id FROM customers c WHERE EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT customer_id FROM customers c WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "columns" sql = '''SELECT order_id, (SELECT COUNT(*) FROM order_items i WHERE i.order_id = o.order_id) AS lines FROM orders o''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(*) FROM orders''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT COUNT(*) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(1) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(order_id) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(DISTINCT customer_id) AS customers FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT SUM(total) AS revenue FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT AVG(total) AS average FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT MIN(total) AS smallest, MAX(total) AS largest FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT MIN(ordered_at) AS first_order, MAX(ordered_at) AS last_order FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT status, COUNT(*) FROM orders GROUP BY status''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY status''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY n DESC''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id HAVING SUM(total) > 1000''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, COUNT(*) AS orders FROM orders GROUP BY customer_id HAVING COUNT(*) > 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, SUM(total) AS revenue, COUNT(*) AS orders, AVG(total) AS aov FROM orders GROUP BY customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT channel, status, COUNT(*) AS n FROM orders GROUP BY channel, status''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT country, COUNT(*) AS customers FROM customers GROUP BY country ORDER BY customers DESC LIMIT 10''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT category, AVG(price) AS avg_price FROM products GROUP BY category''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT category, ROUND(AVG(price), 2) AS avg_price FROM products GROUP BY category''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT category, MEDIAN(price) AS median_price FROM products GROUP BY category''' expect = "refused SQ0008 fn:MEDIAN" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT SUM(quantity * unit_price) AS revenue FROM order_items''' expect = "refused SQ0003 node:mul" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT order_id, SUM(quantity) AS units FROM order_items GROUP BY order_id''' expect = "refused SQ0003 fn:SUM" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT_IF(status = 'cancelled') AS cancelled FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT SUM(IFF(status = 'cancelled', 1, 0)) AS cancelled FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT SUM(CASE WHEN status = 'cancelled' THEN 1 ELSE 0 END) AS cancelled FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders''' expect = "refused SQ0002 fn:COUNT" +expect_repository = "refused SQ0002 fn:COUNT" uncertain = true [[query]] category = "aggregates" sql = '''SELECT ANY_VALUE(status) AS some_status FROM orders''' expect = "refused SQ0008 fn:ANY_VALUE" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, LISTAGG(status, ', ') AS statuses FROM orders GROUP BY customer_id''' expect = "refused SQ0002 fn:LISTAGG" +expect_repository = "refused SQ0002 fn:LISTAGG" [[query]] category = "aggregates" sql = '''SELECT customer_id, LISTAGG(DISTINCT status, ',') WITHIN GROUP (ORDER BY status) AS statuses FROM orders GROUP BY customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, ARRAY_AGG(order_id) AS order_ids FROM orders GROUP BY customer_id''' expect = "refused SQ0008 fn:ARRAY_AGG" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT customer_id, ARRAY_AGG(order_id) WITHIN GROUP (ORDER BY ordered_at) AS order_ids FROM orders GROUP BY customer_id''' expect = "refused SQ0008 fn:ARRAY_AGG" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT STDDEV(total) AS sd, VARIANCE(total) AS var FROM orders''' expect = "refused SQ0002 fn:STDDEV" +expect_repository = "refused SQ0002 fn:STDDEV" [[query]] category = "aggregates" sql = '''SELECT APPROX_COUNT_DISTINCT(customer_id) AS approx_customers FROM orders''' expect = "refused SQ0002 fn:APPROX_COUNT_DISTINCT" +expect_repository = "refused SQ0002 fn:APPROX_COUNT_DISTINCT" [[query]] category = "aggregates" sql = '''SELECT BOOLOR_AGG(is_active) AS any_active FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT MAX_BY(order_id, total) AS biggest_order FROM orders''' expect = "refused SQ0008 fn:MAX_BY" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT MODE(status) AS common_status FROM orders''' expect = "refused SQ0008 fn:MODE" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY total) AS p90 FROM orders''' expect = "refused SQ0008 fn:PERCENTILE_CONT" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT channel, SUM(total) AS revenue FROM orders GROUP BY ROLLUP (channel)''' expect = "refused SQ0002 fn:ROLLUP" +expect_repository = "refused SQ0002 fn:ROLLUP" [[query]] category = "aggregates" sql = '''SELECT channel, status, COUNT(*) AS n FROM orders GROUP BY GROUPING SETS ((channel), (status))''' expect = "refused SQ0002 fn:GROUPING SETS" +expect_repository = "refused SQ0002 fn:GROUPING SETS" [[query]] category = "aggregates" sql = '''SELECT channel, COUNT(*) AS n FROM orders GROUP BY ALL''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month, SUM(total) AS revenue FROM orders GROUP BY 1 ORDER BY 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month, COUNT(DISTINCT customer_id) AS active_customers FROM orders GROUP BY month''' expect = "refused SQ0003 node:group_by" +expect_repository = "refused SQ0003 node:group_by" [[query]] category = "aggregates" sql = '''SELECT YEAR(ordered_at) AS yr, SUM(total) AS revenue FROM orders GROUP BY yr ORDER BY yr''' expect = "refused SQ0003 node:group_by" +expect_repository = "refused SQ0003 node:group_by" [[query]] category = "aggregates" sql = '''SELECT SUM(total) FROM orders''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT MAX(total) FROM orders''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT AVG(total) FROM orders''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT status, SUM(total) FROM orders GROUP BY status''' expect = "refused SQ0003 node:aggregate_function" +expect_repository = "refused SQ0003 node:aggregate_function" [[query]] category = "aggregates" sql = '''SELECT COUNT(*) AS n, SUM(total) / COUNT(*) AS aov FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT SUM(total) / NULLIF(COUNT(*), 0) AS aov FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT DIV0(SUM(total), COUNT(*)) AS aov FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o INNER JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o LEFT JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o LEFT OUTER JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o RIGHT JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o FULL OUTER JOIN customers c ON o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT c.customer_id, o.order_id FROM customers c CROSS JOIN orders o''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o, customers c WHERE o.customer_id = c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c USING (customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT order_id, email FROM orders JOIN customers USING (customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o NATURAL JOIN customers c''' expect = "refused SQ0002 join:natural" +expect_repository = "refused SQ0002 join:natural" [[query]] category = "joins" sql = '''SELECT orders.order_id, customers.email FROM orders JOIN customers ON orders.customer_id = customers.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, i.line_number, p.name FROM orders o JOIN order_items i ON o.order_id = i.order_id JOIN products p ON i.product_id = p.product_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT c.email, o.order_id, i.quantity, p.name, pay.method FROM customers c JOIN orders o ON o.customer_id = c.customer_id JOIN order_items i ON i.order_id = o.order_id JOIN products p ON p.product_id = i.product_id LEFT JOIN payments pay ON pay.order_id = o.order_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT c.customer_id, COUNT(o.order_id) AS orders FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT c.customer_id, COALESCE(SUM(o.total), 0) AS revenue FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id GROUP BY c.customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT p.category, SUM(i.quantity * i.unit_price) AS revenue FROM order_items i JOIN products p ON p.product_id = i.product_id GROUP BY p.category ORDER BY revenue DESC''' expect = "refused SQ0003 node:mul" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT c.customer_id FROM customers c LEFT JOIN orders o ON o.customer_id = c.customer_id WHERE o.order_id IS NULL''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id FROM orders o JOIN payments p ON p.order_id = o.order_id AND p.amount = o.total''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, p.amount FROM orders o LEFT JOIN payments p ON p.order_id = o.order_id AND p.method = 'card'''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT a.order_id, b.order_id AS next_order FROM orders a JOIN orders b ON a.customer_id = b.customer_id AND b.ordered_at > a.ordered_at''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.country FROM orders o JOIN customers c ON o.customer_id = c.customer_id WHERE c.country IN ('US', 'CA') AND o.total > 50''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT t.ticket_id, c.email FROM tickets t JOIN customers c ON c.customer_id = t.customer_id WHERE t.closed_at IS NULL''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, c.email FROM orders o JOIN customers c ON o.customer_id = c.customer_id ORDER BY o.ordered_at DESC LIMIT 20''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, x.total_paid FROM orders o JOIN (SELECT order_id, SUM(amount) AS total_paid FROM payments GROUP BY order_id) x ON x.order_id = o.order_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "joins" sql = '''SELECT o.order_id, x.total_paid FROM orders o LEFT JOIN LATERAL (SELECT SUM(amount) AS total_paid FROM payments p WHERE p.order_id = o.order_id) x''' expect = "refused SQ0002 node:subquery" +expect_repository = "refused SQ0002 node:subquery" [[query]] category = "joins" sql = '''SELECT o.order_id FROM orders o ASOF JOIN payments p MATCH_CONDITION (o.ordered_at >= p.paid_at) ON o.order_id = p.order_id''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "ctes" sql = '''WITH t AS (SELECT 1 AS x) SELECT x FROM t''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH t AS (SELECT 1 AS x) SELECT * FROM t''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH recent AS (SELECT * FROM orders WHERE ordered_at >= '2024-01-01') SELECT COUNT(*) AS n FROM recent''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH shipped AS (SELECT order_id, customer_id, total FROM orders WHERE status = 'shipped') SELECT customer_id, SUM(total) AS revenue FROM shipped GROUP BY customer_id''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH revenue AS (SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id), ranked AS (SELECT customer_id, revenue, RANK() OVER (ORDER BY revenue DESC) AS rnk FROM revenue) SELECT * FROM ranked WHERE rnk <= 10''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH a AS (SELECT customer_id FROM orders), b AS (SELECT customer_id FROM customers) SELECT COUNT(*) AS n FROM a JOIN b USING (customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH totals AS (SELECT order_id, SUM(quantity * unit_price) AS items_total FROM order_items GROUP BY order_id) SELECT o.order_id, o.total, t.items_total, o.total - t.items_total AS diff FROM orders o JOIN totals t ON t.order_id = o.order_id''' expect = "refused SQ0003 node:mul" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH firsts AS (SELECT customer_id, MIN(ordered_at) AS first_order_at FROM orders GROUP BY customer_id) SELECT DATE_TRUNC('month', first_order_at) AS cohort, COUNT(*) AS customers FROM firsts GROUP BY 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH t (a, b) AS (SELECT 1, 2) SELECT a + b AS s FROM t''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "ctes" sql = '''WITH RECURSIVE n AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM n WHERE i < 10) SELECT i FROM n''' expect = "refused SQ0002 node:with" +expect_repository = "refused SQ0002 node:with" [[query]] category = "ctes" sql = '''WITH x AS (SELECT order_id, total FROM orders) SELECT order_id FROM x WHERE total > (SELECT AVG(total) FROM x)''' expect = "translated" +expect_repository = "translated" [[query]] category = "ctes" sql = '''WITH paid AS (SELECT order_id, SUM(amount) AS paid FROM payments GROUP BY order_id) SELECT o.order_id, o.total, COALESCE(p.paid, 0) AS paid, o.total - COALESCE(p.paid, 0) AS outstanding FROM orders o LEFT JOIN paid p ON p.order_id = o.order_id WHERE o.total > COALESCE(p.paid, 0)''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, ROW_NUMBER() OVER (ORDER BY ordered_at) AS rn FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, customer_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) AS order_number FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, RANK() OVER (ORDER BY total DESC) AS rnk FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, DENSE_RANK() OVER (ORDER BY total DESC) AS rnk FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, total, SUM(total) OVER (PARTITION BY customer_id) AS customer_total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, total, SUM(total) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS running_total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, total, SUM(total) OVER (ORDER BY ordered_at ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, AVG(total) OVER (ORDER BY ordered_at ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS moving_avg FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, LAG(total) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS previous_total FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, LEAD(ordered_at) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS next_order_at FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, LAG(total, 1, 0) OVER (ORDER BY ordered_at) AS prev FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, FIRST_VALUE(order_id) OVER (PARTITION BY customer_id ORDER BY ordered_at) AS first_order FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, LAST_VALUE(order_id) OVER (PARTITION BY customer_id ORDER BY ordered_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS last_order FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, NTILE(4) OVER (ORDER BY total) AS quartile FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, total / SUM(total) OVER () AS share FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, RATIO_TO_REPORT(total) OVER () AS share FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, COUNT(*) OVER (PARTITION BY customer_id) AS customer_orders FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, PERCENT_RANK() OVER (ORDER BY total) AS pr FROM orders''' expect = "refused SQ0002 fn:PERCENT_RANK" +expect_repository = "refused SQ0002 fn:PERCENT_RANK" [[query]] category = "windows" sql = '''SELECT order_id, CUME_DIST() OVER (ORDER BY total) AS cd FROM orders''' expect = "refused SQ0002 fn:CUME_DIST" +expect_repository = "refused SQ0002 fn:CUME_DIST" [[query]] category = "windows" sql = '''SELECT * FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) = 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, customer_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) = 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, customer_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) AS rn FROM orders QUALIFY rn = 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT customer_id, order_id FROM orders QUALIFY RANK() OVER (PARTITION BY customer_id ORDER BY total DESC) <= 3''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id FROM (SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at DESC) AS rn FROM orders) WHERE rn = 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, DATEDIFF('day', LAG(ordered_at) OVER (PARTITION BY customer_id ORDER BY ordered_at), ordered_at) AS days_since_previous FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at, order_id) AS seq FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT DATE_TRUNC('day', ordered_at) AS day, SUM(SUM(total)) OVER (ORDER BY DATE_TRUNC('day', ordered_at)) AS cumulative FROM orders GROUP BY 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "windows" sql = '''SELECT order_id, CONDITIONAL_TRUE_EVENT(status = 'cancelled') OVER (ORDER BY ordered_at) AS grp FROM orders''' expect = "refused SQ0002 fn:CONDITIONAL_TRUE_EVENT" +expect_repository = "refused SQ0002 fn:CONDITIONAL_TRUE_EVENT" [[query]] category = "dates" sql = '''SELECT DATE_TRUNC('month', ordered_at) AS month FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE_TRUNC('week', ordered_at) AS week FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE_TRUNC('day', ordered_at) AS day FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE_TRUNC(month, ordered_at) AS month FROM orders''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "dates" sql = '''SELECT ordered_at::DATE AS order_date FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_DATE(ordered_at) AS order_date FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT CAST(ordered_at AS DATE) AS order_date FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE(ordered_at) AS order_date FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEADD(day, 7, ordered_at) AS due_at FROM orders''' expect = "refused SQ0008 fn:DATEADD" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEADD('day', 7, ordered_at) AS due_at FROM orders''' expect = "refused SQ0008 fn:DATEADD" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEADD(month, 1, signup_date) AS next_month FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT ordered_at + INTERVAL '1 day' AS tomorrow FROM orders''' expect = "refused SQ0008 node:add" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT signup_date + 30 AS trial_end FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEDIFF(day, ordered_at, shipped_at) AS days_to_ship FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEDIFF('hour', ordered_at, shipped_at) AS hours_to_ship FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TIMESTAMPDIFF(minute, ordered_at, shipped_at) AS minutes FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT YEAR(ordered_at) AS y, MONTH(ordered_at) AS m, DAY(ordered_at) AS d FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT EXTRACT(year FROM ordered_at) AS y FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE_PART('dow', ordered_at) AS dow FROM orders''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "dates" sql = '''SELECT DAYOFWEEK(ordered_at) AS dow FROM orders''' expect = "refused SQ0002 fn:DAYOFWEEK" +expect_repository = "refused SQ0002 fn:DAYOFWEEK" [[query]] category = "dates" sql = '''SELECT DAYNAME(ordered_at) AS day_name FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT MONTHNAME(ordered_at) AS month_name FROM orders''' expect = "refused SQ0002 fn:MONTHNAME" +expect_repository = "refused SQ0002 fn:MONTHNAME" [[query]] category = "dates" sql = '''SELECT QUARTER(ordered_at) AS q FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT WEEKOFYEAR(ordered_at) AS w FROM orders''' expect = "refused SQ0002 fn:WEEKOFYEAR" +expect_repository = "refused SQ0002 fn:WEEKOFYEAR" [[query]] category = "dates" sql = '''SELECT LAST_DAY(signup_date) AS month_end FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_CHAR(ordered_at, 'YYYY-MM-DD') AS day FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_VARCHAR(ordered_at, 'YYYY-MM') AS month FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_TIMESTAMP('2024-01-15 10:00:00') AS ts''' expect = "refused SQ0008 fn:TO_TIMESTAMP" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_DATE('2024-01-15') AS d''' expect = "refused SQ0008 fn:TO_DATE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TO_DATE('15/01/2024', 'DD/MM/YYYY') AS d''' expect = "refused SQ0008 fn:TO_DATE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TRY_TO_DATE('not a date') AS d''' expect = "refused SQ0008 fn:TRY_TO_DATE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE ordered_at >= '2024-01-01'''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_DATE)''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "refused SQ0003 node:gte" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE ordered_at >= DATEADD(day, -30, CURRENT_TIMESTAMP())''' expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" +expect_repository = "refused SQ0003 fn:DATEADD" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE ordered_at BETWEEN '2024-01-01' AND '2024-01-31'''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE ordered_at::DATE = CURRENT_DATE - 1''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT order_id FROM orders WHERE DATE_TRUNC('month', ordered_at) = DATE_TRUNC('month', CURRENT_DATE)''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "refused SQ0003 node:eq" [[query]] category = "dates" sql = '''SELECT customer_id FROM customers WHERE signup_date > '2023-06-30'::DATE''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATEDIFF(year, signup_date, CURRENT_DATE) AS tenure_years FROM customers''' expect = "refused SQ0008 fn:CURRENT_DATE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT CONVERT_TIMEZONE('UTC', 'America/New_York', ordered_at) AS local_time FROM orders''' expect = "refused SQ0008 fn:CONVERT_TIMEZONE" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT DATE_FROM_PARTS(2024, 1, 15) AS d''' expect = "translated" +expect_repository = "translated" [[query]] category = "dates" sql = '''SELECT TIME_SLICE(ordered_at, 15, 'MINUTE') AS slot FROM orders''' expect = "refused SQ0002 fn:TIME_SLICE" +expect_repository = "refused SQ0002 fn:TIME_SLICE" [[query]] category = "dates" sql = '''SELECT SYSDATE() AS now''' expect = "refused SQ0002 fn:SYSDATE" +expect_repository = "refused SQ0002 fn:SYSDATE" [[query]] category = "dates" sql = '''SELECT GETDATE() AS now''' expect = "refused SQ0002 fn:GETDATE" +expect_repository = "refused SQ0002 fn:GETDATE" [[query]] category = "dates" sql = '''SELECT ADD_MONTHS(signup_date, 3) AS plus3 FROM customers''' expect = "refused SQ0002 fn:ADD_MONTHS" +expect_repository = "refused SQ0002 fn:ADD_MONTHS" [[query]] category = "dates" sql = '''SELECT MONTHS_BETWEEN(CURRENT_DATE, signup_date) AS months FROM customers''' expect = "refused SQ0002 fn:MONTHS_BETWEEN" +expect_repository = "refused SQ0002 fn:MONTHS_BETWEEN" [[query]] category = "dates" sql = '''SELECT order_id, opened_at FROM tickets t JOIN orders o ON DATE_TRUNC('day', t.opened_at) = DATE_TRUNC('day', o.ordered_at)''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, IFF(total > 100, 'large', 'small') AS size FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, CASE WHEN total > 100 THEN 'large' ELSE 'small' END AS size FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, CASE WHEN total > 500 THEN 'xl' WHEN total > 100 THEN 'l' WHEN total > 20 THEN 'm' ELSE 's' END AS size FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, CASE status WHEN 'shipped' THEN 1 WHEN 'delivered' THEN 2 ELSE 0 END AS stage FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, CASE WHEN discount IS NULL THEN 0 ELSE discount END AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, COALESCE(discount, 0) AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, NVL(discount, 0) AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, IFNULL(discount, 0) AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, ZEROIFNULL(discount) AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, NULLIF(discount, 0) AS discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, NVL2(shipped_at, 'shipped', 'pending') AS state FROM orders''' expect = "refused SQ0002 fn:NVL2" +expect_repository = "refused SQ0002 fn:NVL2" [[query]] category = "conditionals" sql = '''SELECT order_id, COALESCE(shipped_at, ordered_at) AS last_event_at FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, GREATEST(total, discount) AS g, LEAST(total, discount) AS l FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, DECODE(status, 'shipped', 1, 'delivered', 2, 0) AS stage FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, IFF(shipped_at IS NULL, NULL, DATEDIFF(day, ordered_at, shipped_at)) AS days FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT customer_id, IFF(is_active, 'active', 'inactive') AS state FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT customer_id, CASE WHEN country IN ('DE', 'FR', 'IT') THEN 'EU' WHEN country = 'US' THEN 'NA' ELSE 'Other' END AS region FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, total > 100 AS is_large FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, EQUAL_NULL(discount, NULL) AS no_discount FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, discount IS DISTINCT FROM 0 AS has_discount FROM orders''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "conditionals" sql = '''SELECT order_id, IFF(total > 100, total * 0.9, total) AS charged FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "conditionals" sql = '''SELECT order_id, CASE WHEN total >= 100 AND channel = 'web' THEN 'promo' END AS promo FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT UPPER(email) AS e FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LOWER(email) AS e FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT TRIM(first_name) AS f FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LTRIM(first_name) AS f, RTRIM(last_name) AS l FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LENGTH(email) AS n FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LEN(email) AS n FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT SUBSTR(email, 1, 5) AS prefix FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT SUBSTRING(email, 1, 5) AS prefix FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LEFT(sku, 3) AS family, RIGHT(sku, 4) AS suffix FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT SPLIT_PART(email, '@', 2) AS domain FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT SPLIT_PART(email, '@', 1) AS local_part FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT POSITION('@' IN email) AS at_pos FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT CHARINDEX('@', email) AS at_pos FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT REPLACE(sku, '-', '') AS clean_sku FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT REGEXP_REPLACE(sku, '[^A-Z0-9]', '') AS clean_sku FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT REGEXP_SUBSTR(email, '@(.*)$', 1, 1, 'e') AS domain FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT REGEXP_LIKE(email, '.*@example\\.com') AS is_example FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT INITCAP(first_name) AS f FROM customers''' expect = "refused SQ0008 fn:INITCAP" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT CONCAT_WS(' ', first_name, last_name) AS full_name FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT LPAD(CAST(order_id AS VARCHAR), 8, '0') AS padded FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT order_id::VARCHAR AS id_text FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT TO_VARCHAR(order_id) AS id_text FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT TO_VARCHAR(total, '999,999.00') AS formatted FROM orders''' expect = "refused SQ0003 fn:TO_VARCHAR" +expect_repository = "refused SQ0003 fn:TO_VARCHAR" [[query]] category = "strings" sql = '''SELECT TO_NUMBER('123.45', 10, 2) AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT TRY_TO_NUMBER('abc') AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT TRY_CAST('abc' AS INTEGER) AS n''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT STARTSWITH(email, 'admin') AS is_admin FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT ENDSWITH(email, '.com') AS dot_com FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT CONTAINS(email, 'test') AS is_test FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT REVERSE(sku) AS r FROM products''' expect = "refused SQ0002 fn:REVERSE" +expect_repository = "refused SQ0002 fn:REVERSE" [[query]] category = "strings" sql = '''SELECT REPEAT('-', 10) AS line''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT MD5(email) AS hashed FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT SHA2(email, 256) AS hashed FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT HASH(email) AS h FROM customers''' expect = "refused SQ0008 fn:HASH" +expect_repository = "refused SQ0008 fn:HASH" [[query]] category = "strings" sql = '''SELECT SPLIT(email, '@') AS parts FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT STRTOK(email, '@', 1) AS local_part FROM customers''' expect = "refused SQ0002 fn:STRTOK" +expect_repository = "refused SQ0002 fn:STRTOK" [[query]] category = "strings" sql = '''SELECT ASCII(first_name) AS code FROM customers''' expect = "refused SQ0002 fn:ASCII" +expect_repository = "refused SQ0002 fn:ASCII" [[query]] category = "strings" sql = '''SELECT UUID_STRING() AS id''' expect = "refused SQ0002 fn:UUID_STRING" +expect_repository = "refused SQ0002 fn:UUID_STRING" [[query]] category = "strings" sql = '''SELECT email FROM customers WHERE LOWER(email) = 'a@example.com'''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT email FROM customers WHERE TRIM(LOWER(email)) LIKE '%example%'''' expect = "translated" +expect_repository = "translated" [[query]] category = "strings" sql = '''SELECT COLLATE(email, 'en-ci') AS e FROM customers''' expect = "refused SQ0002 fn:COLLATE" +expect_repository = "refused SQ0002 fn:COLLATE" uncertain = true [[query]] category = "strings" sql = '''SELECT EDITDISTANCE(first_name, last_name) AS d FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT ROUND(total) AS r FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT ROUND(total, 1) AS r FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT FLOOR(total) AS f, CEIL(total) AS c FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT TRUNC(total) AS t FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT TRUNCATE(total, 1) AS t FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT MOD(order_id, 10) AS bucket FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT order_id % 10 AS bucket FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT POWER(total, 2) AS sq FROM orders''' expect = "refused SQ0003 fn:POWER" +expect_repository = "refused SQ0003 fn:POWER" [[query]] category = "numeric" sql = '''SELECT SQRT(total) AS root FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT LN(total) AS l FROM orders WHERE total > 0''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT LOG(10, total) AS l FROM orders WHERE total > 0''' expect = "refused SQ0002 fn:LOG" +expect_repository = "refused SQ0002 fn:LOG" [[query]] category = "numeric" sql = '''SELECT EXP(1) AS e''' expect = "refused SQ0008 fn:EXP" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT SIGN(total - 100) AS s FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT total::FLOAT AS f FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT total::NUMBER(10,0) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT CAST(total AS DECIMAL(10, 1)) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT CAST(total AS INT) AS n FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT weight_kg * 1000 AS grams FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT ROUND(weight_kg, 2) AS w FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT price * 1.0 / 3 AS third FROM products''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT quantity * unit_price AS line_total FROM order_items''' expect = "refused SQ0003 node:mul" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT SUM(quantity) * 1.0 / COUNT(DISTINCT order_id) AS units_per_order FROM order_items''' expect = "refused SQ0003 fn:SUM" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT UNIFORM(1, 10, RANDOM()) AS r''' expect = "refused SQ0002 fn:UNIFORM" +expect_repository = "refused SQ0002 fn:UNIFORM" [[query]] category = "numeric" sql = '''SELECT RANDOM() AS r''' expect = "refused SQ0002 fn:RANDOM" +expect_repository = "refused SQ0002 fn:RANDOM" [[query]] category = "numeric" sql = '''SELECT order_id FROM orders WHERE total::INT = 100''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT BITAND(order_id, 1) AS odd FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "numeric" sql = '''SELECT WIDTH_BUCKET(total, 0, 1000, 10) AS bucket FROM orders''' expect = "refused SQ0002 fn:WIDTH_BUCKET" +expect_repository = "refused SQ0002 fn:WIDTH_BUCKET" [[query]] category = "semi_structured" sql = '''SELECT payload:page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "refused SQ0003 node:j_s_o_n_extract" [[query]] category = "semi_structured" sql = '''SELECT payload:page AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:page::STRING AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:page::VARCHAR AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:device.os::STRING AS os FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:items[0].sku::STRING AS first_sku FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload['page']::STRING AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:"page"::STRING AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:amount::NUMBER(10,2) AS amount FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:amount::FLOAT AS amount FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT event_id FROM events WHERE payload:page::STRING = '/checkout'''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:page::STRING AS page, COUNT(*) AS views FROM events WHERE event_type = 'page_view' GROUP BY 1 ORDER BY 2 DESC''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT GET(payload, 'page') AS page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT GET_PATH(payload, 'device.os') AS os FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT TRY_PARSE_JSON('{"a": 1}') AS j''' expect = "refused SQ0008 fn:TRY_PARSE_JSON" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT PARSE_JSON('{"a": 1}'):a::INT AS a''' expect = "refused SQ0008 fn:PARSE_JSON" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT OBJECT_CONSTRUCT('id', order_id, 'total', total) AS o FROM orders''' expect = "refused SQ0008 fn:OBJECT_CONSTRUCT" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ARRAY_CONSTRUCT(1, 2, 3) AS arr''' expect = "refused SQ0008 fn:ARRAY_CONSTRUCT" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ARRAY_SIZE(tags) AS n FROM tickets''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ARRAY_CONTAINS('urgent'::VARIANT, tags) AS urgent FROM tickets''' expect = "refused SQ0008 node:column" +expect_repository = "translated" uncertain = true [[query]] category = "semi_structured" sql = '''SELECT tags[0]::STRING AS first_tag FROM tickets''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ticket_id, f.value::STRING AS tag FROM tickets, LATERAL FLATTEN(input => tags) f''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ticket_id, f.value::STRING AS tag FROM tickets t, TABLE(FLATTEN(t.tags)) f''' expect = "refused SQ0008 node:column" +expect_repository = "refused SQ0002 fn:FLATTEN" [[query]] category = "semi_structured" sql = '''SELECT e.event_id, i.value:sku::STRING AS sku, i.value:qty::INT AS qty FROM events e, LATERAL FLATTEN(input => e.payload:items) i''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT e.event_id, i.index, i.value FROM events e, LATERAL FLATTEN(input => e.payload:items) i''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT event_id, OBJECT_KEYS(payload) AS keys FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT TYPEOF(payload) AS t FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT IS_NULL_VALUE(payload:page) AS null_page FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload:page IS NULL AS missing FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT event_id FROM events WHERE payload:amount::NUMBER > 100''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT TO_JSON(payload) AS txt FROM events''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT payload FROM events LIMIT 10''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT ARRAY_TO_STRING(tags, ',') AS tag_list FROM tickets''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT customer_id, ARRAY_AGG(DISTINCT event_type) AS types FROM events GROUP BY customer_id''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "semi_structured" sql = '''SELECT customer_id, OBJECT_AGG(event_type, payload) AS by_type FROM events GROUP BY customer_id''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -1971,6 +2360,7 @@ renamed as ( ) select * from renamed''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -1992,6 +2382,7 @@ final as ( ) select * from final''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2017,6 +2408,7 @@ select from customers left join customer_orders using (customer_id)''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2031,6 +2423,7 @@ from orders where status != 'cancelled' group by 1, 2''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2043,6 +2436,7 @@ with events_ranked as ( ) select * exclude (rn) from events_ranked where rn = 1''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2058,6 +2452,7 @@ select datediff('day', ordered_at, coalesce(shipped_at, current_timestamp())) as age_days from orders''' expect = "refused SQ0008 fn:CURRENT_TIMESTAMP" +expect_repository = "refused SQ0003 fn:DATEDIFF" [[query]] category = "dbt_models" @@ -2073,6 +2468,7 @@ select avg(revenue) over (order by day rows between 6 preceding and current row) as revenue_7d_avg from daily''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2088,6 +2484,7 @@ from products p left join order_items i on i.product_id = p.product_id group by 1, 2, 3''' expect = "refused SQ0003 fn:SUM" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2104,6 +2501,7 @@ select priority, avg(minutes_open) as avg_minutes, max(minutes_open) as max_minu from ticket_durations group by priority''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2116,6 +2514,7 @@ select from events group by customer_id''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2133,6 +2532,7 @@ select revenue - lag(revenue) over (partition by customer_id order by month) as change from monthly''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2142,6 +2542,7 @@ select order_id from {{ ref('stg_order_items') }}''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "dbt_models" @@ -2152,6 +2553,7 @@ select line_number from order_items''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2160,6 +2562,7 @@ select * from orders where ordered_at > (select coalesce(max(ordered_at), '1900-01-01') from orders)''' expect = "refused SQ0003 fn:COALESCE" +expect_repository = "refused SQ0003 fn:COALESCE" [[query]] category = "dbt_models" @@ -2173,6 +2576,7 @@ from spine left join orders o on o.ordered_at::date = spine.day group by 1''' expect = "translated" +expect_repository = "translated" [[query]] category = "dbt_models" @@ -2188,57 +2592,68 @@ from customers where email is not null and country is not null''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id FROM orders UNION SELECT customer_id FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id FROM orders UNION ALL SELECT customer_id FROM tickets''' expect = "refused SQ0008 node:column" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id FROM customers EXCEPT SELECT customer_id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id FROM customers MINUS SELECT customer_id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id FROM customers INTERSECT SELECT customer_id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT 'orders' AS source, COUNT(*) AS n FROM orders UNION ALL SELECT 'customers', COUNT(*) FROM customers''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT order_id, total FROM orders WHERE total > 100 UNION ALL SELECT order_id, total FROM orders WHERE total < 10 ORDER BY total''' expect = "translated" +expect_repository = "translated" [[query]] category = "set_operations" sql = '''SELECT customer_id, email FROM customers UNION ALL BY NAME SELECT email, customer_id FROM customers''' expect = "translated" +expect_repository = "translated" uncertain = true [[query]] category = "misc" sql = '''SELECT order_id FROM orders WHERE order_id = 1;''' expect = "translated" +expect_repository = "translated" [[query]] category = "misc" sql = '''SELECT 1; SELECT 2''' expect = "refused SQ0003 node:literal" +expect_repository = "refused SQ0003 node:literal" [[query]] category = "misc" @@ -2246,505 +2661,605 @@ sql = ''' SELECT order_id -- the id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "misc" sql = '''/* comment */ SELECT order_id FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "misc" sql = '''SELECT COUNT(*) AS n FROM orders WHERE status ILIKE ANY ('ship%', 'deliv%')''' expect = "refused SQ0008 node:i_like" +expect_repository = "translated" [[query]] category = "misc" sql = '''SELECT order_id FROM orders WHERE status LIKE ANY ('ship%', 'deliv%')''' expect = "refused SQ0003 node:like" +expect_repository = "refused SQ0003 node:like" [[query]] category = "misc" sql = '''SELECT order_id FROM orders ORDER BY RANDOM() LIMIT 1''' expect = "refused SQ0002 fn:RANDOM" +expect_repository = "refused SQ0002 fn:RANDOM" [[query]] category = "misc" sql = '''SELECT order_id FROM orders WHERE (customer_id, status) IN (SELECT customer_id, 'shipped' FROM customers)''' expect = "refused SQ0002 node:tuple" +expect_repository = "refused SQ0002 node:tuple" uncertain = true [[query]] category = "misc" sql = '''SELECT o.order_id FROM orders o WHERE o.total > ALL (SELECT amount FROM payments)''' expect = "refused SQ0002 node:all" +expect_repository = "refused SQ0002 node:all" [[query]] category = "misc" sql = '''SELECT IDENTIFIER('orders') AS x''' expect = "refused SQ0002 fn:IDENTIFIER" +expect_repository = "refused SQ0002 fn:IDENTIFIER" [[query]] category = "misc" sql = '''SELECT * FROM IDENTIFIER('orders')''' expect = "refused SQ0002 node:table_function" +expect_repository = "refused SQ0002 node:table_function" [[query]] category = "misc" sql = '''SELECT CURRENT_USER() AS u''' expect = "refused SQ0002 fn:CURRENT_USER" +expect_repository = "refused SQ0002 fn:CURRENT_USER" [[query]] category = "misc" sql = '''SELECT CURRENT_WAREHOUSE() AS w''' expect = "refused SQ0002 fn:CURRENT_WAREHOUSE" +expect_repository = "refused SQ0002 fn:CURRENT_WAREHOUSE" [[query]] category = "misc" sql = '''SELECT $1 FROM orders''' expect = "refused SQ0003 node:parameter" +expect_repository = "refused SQ0003 node:parameter" [[query]] category = "misc" sql = '''SELECT order_id FROM orders AT(OFFSET => -60)''' expect = "refused SQ0002 node:table" +expect_repository = "refused SQ0002 node:table" [[query]] category = "misc" sql = '''SELECT order_id FROM orders WHERE customer_id = :customer_id''' expect = "refused SQ0002 node:parameter" +expect_repository = "refused SQ0002 node:parameter" [[query]] category = "misc" sql = '''SELECT order_id FROM orders WHERE customer_id = ?''' expect = "refused SQ0002 node:placeholder" +expect_repository = "refused SQ0002 node:placeholder" [[query]] category = "misc" sql = '''SELECT SEQ4() AS n FROM TABLE(GENERATOR(ROWCOUNT => 5))''' expect = "translated" +expect_repository = "translated" [[query]] category = "misc" sql = '''SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p''' expect = "refused SQ0003 node:pivot" +expect_repository = "refused SQ0003 node:pivot" uncertain = true [[query]] category = "misc" sql = '''SELECT * FROM (SELECT channel, status, total FROM orders) PIVOT (SUM(total) FOR status IN ('shipped', 'pending'))''' expect = "refused SQ0003 node:pivot" +expect_repository = "refused SQ0003 node:pivot" [[query]] category = "misc" sql = '''SELECT order_id, HASH_AGG(*) AS h FROM orders GROUP BY order_id''' expect = "refused SQ0002 fn:HASH_AGG" +expect_repository = "refused SQ0002 fn:HASH_AGG" uncertain = true [[query]] category = "misc" sql = '''SELECT CURRENT_ACCOUNT() AS a''' expect = "refused SQ0002 fn:CURRENT_ACCOUNT" +expect_repository = "refused SQ0002 fn:CURRENT_ACCOUNT" [[query]] category = "ddl_dml" sql = '''CREATE TABLE big_orders AS SELECT * FROM orders WHERE total > 1000''' expect = "refused SQ0002 node:create_table" +expect_repository = "refused SQ0002 node:create_table" [[query]] category = "ddl_dml" sql = '''CREATE OR REPLACE TABLE big_orders AS SELECT order_id, total FROM orders WHERE total > 1000''' expect = "refused SQ0002 node:create_table" +expect_repository = "refused SQ0002 node:create_table" [[query]] category = "ddl_dml" sql = '''CREATE VIEW v_orders AS SELECT order_id, total FROM orders''' expect = "refused SQ0002 node:create_view" +expect_repository = "refused SQ0002 node:create_view" [[query]] category = "ddl_dml" sql = '''CREATE TEMPORARY TABLE t AS SELECT 1 AS x''' expect = "refused SQ0002 node:create_table" +expect_repository = "refused SQ0002 node:create_table" [[query]] category = "ddl_dml" sql = '''INSERT INTO orders (order_id, customer_id, status, total) VALUES (1, 1, 'new', 10.00)''' expect = "refused SQ0002 node:insert" +expect_repository = "refused SQ0002 node:insert" [[query]] category = "ddl_dml" sql = '''INSERT INTO orders SELECT * FROM orders WHERE order_id = 1''' expect = "refused SQ0002 node:insert" +expect_repository = "refused SQ0002 node:insert" [[query]] category = "ddl_dml" sql = '''UPDATE orders SET status = 'cancelled' WHERE order_id = 1''' expect = "refused SQ0002 node:update" +expect_repository = "refused SQ0002 node:update" [[query]] category = "ddl_dml" sql = '''DELETE FROM orders WHERE status = 'cancelled'''' expect = "refused SQ0002 node:delete" +expect_repository = "refused SQ0002 node:delete" [[query]] category = "ddl_dml" sql = '''MERGE INTO orders t USING orders s ON t.order_id = s.order_id WHEN MATCHED THEN UPDATE SET t.status = s.status WHEN NOT MATCHED THEN INSERT (order_id, status) VALUES (s.order_id, s.status)''' expect = "refused SQ0002 node:merge" +expect_repository = "refused SQ0002 node:merge" [[query]] category = "ddl_dml" sql = '''TRUNCATE TABLE orders''' expect = "refused SQ0002 node:truncate" +expect_repository = "refused SQ0002 node:truncate" [[query]] category = "ddl_dml" sql = '''DROP TABLE IF EXISTS big_orders''' expect = "refused SQ0002 node:drop_table" +expect_repository = "refused SQ0002 node:drop_table" [[query]] category = "ddl_dml" sql = '''CREATE TABLE t (id INT, name VARCHAR)''' expect = "refused SQ0002 node:create_table" +expect_repository = "refused SQ0002 node:create_table" [[query]] category = "ddl_dml" sql = '''ALTER TABLE orders ADD COLUMN note VARCHAR''' expect = "refused SQ0002 node:alter_table" +expect_repository = "refused SQ0002 node:alter_table" [[query]] category = "ddl_dml" sql = '''SHOW TABLES''' expect = "refused SQ0002 node:show" +expect_repository = "refused SQ0002 node:show" [[query]] category = "ddl_dml" sql = '''DESCRIBE TABLE orders''' expect = "refused SQ0002 node:describe" +expect_repository = "refused SQ0002 node:describe" [[query]] category = "ddl_dml" sql = '''USE WAREHOUSE compute_wh''' expect = "refused SQ0002 node:use" +expect_repository = "refused SQ0002 node:use" [[query]] category = "invalid_syntax" sql = '''SELEC 1''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT * FORM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT * FROM''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id total FROM orders WHERE''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id, FROM orders''' expect = "translated" +expect_repository = "translated" [[query]] category = "invalid_syntax" sql = '''SELECT order_id,, total FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total >''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status = 'shipped''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE (total > 10''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total > 10)''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders GROUP status''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders ORDER total''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders LIMIT''' expect = "refused SQ0003 node:limit" +expect_repository = "refused SQ0003 node:limit" [[query]] category = "invalid_syntax" sql = '''SELECT COUNT(* FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT SUM(total)) FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders o JOIN customers c''' expect = "refused SQ0002 join:inner" +expect_repository = "refused SQ0002 join:inner" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders o JOIN customers c ON''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status == 'shipped'''' expect = "translated" +expect_repository = "translated" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status = "shipped"''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_syntax" sql = '''SELECT CASE WHEN total > 1 THEN 'a' FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT CASE total > 1 THEN 'a' END FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total BETWEEN 1''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status IN 'shipped'''' expect = "refused SQ0002 node:in" +expect_repository = "refused SQ0002 node:in" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status IN ()''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''WITH t AS SELECT 1 SELECT * FROM t''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''WITH t AS (SELECT 1) SELECT * FROM''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders UNION''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT * FROM orders WHERE''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders HAVING''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT DISTINCT FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT 1 +''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT 'unterminated''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders;;''' expect = "translated" +expect_repository = "translated" [[query]] category = "invalid_syntax" sql = '''SELECT order_id AS FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total >> 10''' expect = "refused SQ0002 node:bitwise_right_shift" +expect_repository = "refused SQ0002 node:bitwise_right_shift" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders o LEFT orders c ON 1 = 1''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT row_number() OVER (PARTITION customer_id) FROM orders''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders QUALIFY''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT IFF(total > 10, 'a') AS x FROM orders''' expect = "refused SQ0002 fn:IFF" +expect_repository = "refused SQ0002 fn:IFF" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE order_id IN (1, 2,)''' expect = "parse SQ0001" +expect_repository = "parse SQ0001" [[query]] category = "invalid_semantics" sql = '''SELECT discount_code FROM orders''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM order''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orderz''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT o.order_id FROM orders''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT x.order_id FROM orders o''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT custmer_id FROM customers''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id, status FROM orders GROUP BY status''' expect = "refused SQ0002 node:statement" +expect_repository = "refused SQ0002 node:statement" [[query]] category = "invalid_semantics" sql = '''SELECT status, total FROM orders GROUP BY status''' expect = "refused SQ0002 node:statement" +expect_repository = "refused SQ0002 node:statement" [[query]] category = "invalid_semantics" sql = '''SELECT customer_id FROM orders o JOIN customers c ON o.customer_id = c.customer_id''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders WHERE SUM(total) > 10''' expect = "refused SQ0003 fn:SUM" +expect_repository = "refused SQ0003 fn:SUM" [[query]] category = "invalid_semantics" sql = '''SELECT COUNT(*) AS n FROM orders WHERE n > 1''' expect = "refused SQ0002 node:statement" +expect_repository = "refused SQ0002 node:statement" [[query]] category = "invalid_semantics" sql = '''SELECT SUM(SUM(total)) AS s FROM orders''' expect = "refused SQ0003 fn:SUM" +expect_repository = "refused SQ0003 fn:SUM" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders ORDER BY 5''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders GROUP BY 3''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders WHERE ROW_NUMBER() OVER (ORDER BY total) = 1''' expect = "refused SQ0003 node:window_function" +expect_repository = "refused SQ0003 node:window_function" [[query]] category = "invalid_semantics" sql = '''SELECT order_id, total + 'abc' AS x FROM orders''' expect = "refused SQ0003 node:add" +expect_repository = "refused SQ0003 node:add" uncertain = true [[query]] category = "invalid_semantics" sql = '''SELECT DATEADD(day, 'x', ordered_at) AS x FROM orders''' expect = "refused SQ0003 fn:DATEADD" +expect_repository = "refused SQ0003 fn:DATEADD" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders WHERE ordered_at > 5''' expect = "refused SQ0003 node:gt" +expect_repository = "refused SQ0003 node:gt" uncertain = true [[query]] category = "invalid_semantics" sql = '''SELECT LENGTH(order_id, 2) AS x FROM orders''' expect = "refused SQ0002 fn:LENGTH" +expect_repository = "refused SQ0002 fn:LENGTH" uncertain = true [[query]] category = "invalid_semantics" sql = '''SELECT NOSUCHFUNCTION(order_id) AS x FROM orders''' expect = "refused SQ0002 fn:NOSUCHFUNCTION" +expect_repository = "refused SQ0002 fn:NOSUCHFUNCTION" [[query]] category = "invalid_semantics" sql = '''SELECT 1 QUALIFY TRUE''' expect = "check SQ0203" +expect_repository = "check SQ0203" [[query]] category = "invalid_semantics" sql = '''SELECT customer_id, SUM(total) AS revenue FROM orders GROUP BY customer_id QUALIFY revenue > 100''' expect = "check SQ0203" +expect_repository = "check SQ0203" [[query]] category = "invalid_semantics" sql = '''SELECT payload:page FROM orders''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders o, orders o''' expect = "refused SQ0004 statement" +expect_repository = "refused SQ0004 statement" uncertain = true diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index 77adbe9c7..6f5b650fa 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -30,7 +30,7 @@ use crate::analysis::main::confirm_assumption::confirm_assumption; use crate::analysis::main::refused_construct::refused_construct; use crate::analysis::models::PipelineConverter; use crate::analysis::tests::test_types::{ - AnalysisSummary, CollectedSummary, DivergenceModeOutcome, EverydayCorpus, + AnalysisSummary, CollectedSummary, DivergenceModeOutcome, EverydayCorpus, EverydayQuery, }; use crate::cli::main::convert_report::convert_report; use crate::cli::models::{ConvertArguments, OutputArguments, OutputFormat, PolicyArguments}; @@ -783,13 +783,76 @@ const PARSED_TREE: &str = r#"{"error":false"#; /// The start of DuckDB's source-offset fields, `query_location` and `query_location_length`. const LOCATION_FIELD: &str = r#""query_location"#; -/// Each everyday corpus query whose outcome differs from its recorded `expect`, with both. +/// Each everyday corpus query whose outcome under either policy differs from the recorded one. pub(crate) fn everyday_outcome_mismatches() -> Vec { + let (root, corpus) = everyday_corpus(); + let strict = everyday_arguments(&root, true); + let repository = everyday_arguments(&root, false); + corpus + .query + .iter() + .flat_map(|query| { + [ + ( + "strict", + &query.expect, + everyday_outcome(&query.sql, &strict), + ), + ( + "repository", + &query.expect_repository, + everyday_outcome(&query.sql, &repository), + ), + ] + .into_iter() + .filter(|(_, expected, outcome)| outcome != *expected) + .map(|(policy, expected, outcome)| { + format!( + "{} under {policy}: expected {expected}, got {outcome}", + query.sql + ) + }) + .collect::>() + }) + .collect() +} + +/// How many valid queries, not DDL, are recorded as translated: strict, and repository policy. +pub(crate) fn everyday_translation_rates() -> String { + let (_, corpus) = everyday_corpus(); + let valid: Vec<&EverydayQuery> = corpus + .query + .iter() + .filter(|query| !EVERYDAY_NOT_TRANSLATABLE.contains(&query.category.as_str())) + .filter(|query| !query.category.starts_with(EVERYDAY_INVALID_PREFIX)) + .collect(); + let strict = valid + .iter() + .filter(|query| query.expect == TRANSLATED) + .count(); + let repository = valid + .iter() + .filter(|query| query.expect_repository == TRANSLATED) + .count(); + format!( + "{strict} of {} strict, {repository} of {} repository", + valid.len(), + valid.len() + ) +} + +/// The everyday corpus directory and its queries. +fn everyday_corpus() -> (std::path::PathBuf, EverydayCorpus) { let root = repository_root().join(EVERYDAY_CORPUS); let text = std::fs::read_to_string(root.join("queries.toml")).expect("the corpus reads"); let corpus: EverydayCorpus = toml::from_str(&text).expect("the corpus parses"); assert!(corpus.query.len() > 400, "{} queries", corpus.query.len()); - let arguments = ConvertArguments { + (root, corpus) +} + +/// `convert`'s arguments: the built-in policy, or the repository's under a UTC session. +fn everyday_arguments(root: &std::path::Path, builtin_policy: bool) -> ConvertArguments { + ConvertArguments { path: None, schema: Some(root.join("schema.json")), overrides: None, @@ -801,22 +864,15 @@ pub(crate) fn everyday_outcome_mismatches() -> Vec { json: false, }, conversion: PolicyArguments { - policy: POLICY_FILE_NAME.into(), - builtin_policy: true, + policy: repository_root().join(POLICY_FILE_NAME), + builtin_policy, strict: false, - utc_session: false, + utc_session: !builtin_policy, confirm_declared_offsets: false, python_runtime: false, loud_divergences_only: false, }, - }; - corpus - .query - .iter() - .map(|query| (query, everyday_outcome(&query.sql, &arguments))) - .filter(|(query, outcome)| *outcome != query.expect) - .map(|(query, outcome)| format!("{}: expected {}, got {outcome}", query.sql, query.expect)) - .collect() + } } /// What SQLQuack does with `sql`: its first check diagnostic, first refusal, or `translated`. @@ -843,6 +899,12 @@ fn everyday_outcome(sql: &str, arguments: &ConvertArguments) -> String { /// The corpus directory, under the repository root. const EVERYDAY_CORPUS: &str = "corpora/everyday"; +/// Corpus categories of statements no rule translates: DDL and DML. +const EVERYDAY_NOT_TRANSLATABLE: [&str; 1] = ["ddl_dml"]; + +/// The prefix of corpus categories of SQL Snowflake rejects. +const EVERYDAY_INVALID_PREFIX: &str = "invalid"; + /// The outcome of a query that converts. const TRANSLATED: &str = "translated"; diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index ee68a37eb..095538b08 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -1,11 +1,12 @@ -use crate::analysis::tests::helpers::everyday_outcome_mismatches; +use crate::analysis::tests::helpers::{everyday_outcome_mismatches, everyday_translation_rates}; use crate::analysis::tests::test_types::EverydayCorpusTestCase; #[test] fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_its_outcome() { let test_cases = [EverydayCorpusTestCase { - description: "every corpora/everyday query parses, checks, converts or refuses as recorded", + description: "every corpora/everyday query parses, checks, converts or refuses as recorded, under the strict built-in policy and the repository policy", expected_mismatches: Vec::new(), + expected_rates: "285 of 434 strict, 354 of 434 repository", }]; for test_case in test_cases { @@ -15,5 +16,11 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i "{}", test_case.description ); + assert_eq!( + everyday_translation_rates(), + test_case.expected_rates, + "{}", + test_case.description + ); } } diff --git a/crates/sqlquack/src/analysis/tests/test_types.rs b/crates/sqlquack/src/analysis/tests/test_types.rs index f0268816f..1696d38f9 100644 --- a/crates/sqlquack/src/analysis/tests/test_types.rs +++ b/crates/sqlquack/src/analysis/tests/test_types.rs @@ -214,6 +214,7 @@ pub(crate) struct OverrideCallsTestCase { pub(crate) struct EverydayCorpusTestCase { pub(crate) description: &'static str, pub(crate) expected_mismatches: Vec, + pub(crate) expected_rates: &'static str, } /// The everyday corpus file. @@ -225,6 +226,8 @@ pub(crate) struct EverydayCorpus { /// One everyday query and the outcome recorded for it. #[derive(serde::Deserialize)] pub(crate) struct EverydayQuery { + pub(crate) category: String, pub(crate) sql: String, pub(crate) expect: String, + pub(crate) expect_repository: String, } diff --git a/website/public/playground/index.html b/website/public/playground/index.html index 9337148ff..2fcc279e0 100644 --- a/website/public/playground/index.html +++ b/website/public/playground/index.html @@ -92,7 +92,7 @@

Playground

Policy and session - +

A policy file, as for --policy.

Snowflake session parameters, one NAME=VALUE per line, as for --session.

@@ -110,7 +110,7 @@

Playground

-
3

Translate to DuckDB

…
+
3

Translate to DuckDB

…
diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index ae049506b..ff1099098 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -34,6 +34,28 @@ const strictBox = $('#pg-strict'); const utcBox = $('#pg-utc'); const settings = () => JSON.stringify({ policy: policyBox.value, session: sessionBox.value, strict: strictBox.checked, utc_session: utcBox.checked }); +const modeButton = $('#pg-policy-mode'); +// The repository's sqlquack.toml, which the playground starts with, under a UTC session. +let repositoryPolicy = ''; +const policyMode = () => { + const plain = !strictBox.checked && !sessionBox.value.trim(); + if (plain && utcBox.checked && policyBox.value === repositoryPolicy) return 'repository'; + if (plain && !utcBox.checked && !policyBox.value.trim()) return 'strict'; + return 'custom'; +}; +const POLICY_LABELS = { repository: 'policy: sqlquack.toml · UTC session', strict: 'policy: strict (built-in)' }; +function showPolicy() { + const mode = policyMode(); + $('#stage-translate .pg-sub').textContent = POLICY_LABELS[mode] ?? 'policy: custom'; + modeButton.textContent = mode === 'repository' ? 'Use strict' : 'Use sqlquack.toml'; + modeButton.title = mode === 'repository' ? 'Switch to the built-in policy, which refuses every known divergence' : 'Switch to the repository policy under a UTC session'; +} +function usePolicy(mode) { + policyBox.value = mode === 'repository' ? repositoryPolicy : ''; + utcBox.checked = mode === 'repository'; + strictBox.checked = false; + sessionBox.value = ''; +} const caption = $('#pg-caption'); const verdict = $('#pg-verdict'); const marks = $('#pg-marks'); @@ -70,6 +92,7 @@ let current = 0; async function run() { const ticket = ++current; const sql = sqlBox.value; + showPolicy(); const reply = await engine('analyze', { sql, schema: schemaBox.value, settings: settings() }); // A newer edit has already been sent; its reply will render instead. if (ticket !== current) return; @@ -129,7 +152,8 @@ async function run() { if (sql) body = `
${terminal(escape(sql))}
`; body += (translate.diagnostics ?? []).map((d) => terminal(tone(rendered(d, chars)))).join(''); } - const policy = translate.profile ? `policy: ${translate.profile}` : ''; + const mode = policyMode(); + const policy = POLICY_LABELS[mode] ?? `policy: custom${translate.profile ? ` · ${translate.profile}` : ''}`; if (refused) { const reason = translate.diagnostics?.[0]?.message ?? 'no supported conversion'; verdict.innerHTML = `Valid Snowflake SQL. Not translated to DuckDB: ${escape(reason)}`; @@ -219,10 +243,14 @@ sqlBox.addEventListener('input', () => { }); schemaBox.addEventListener('input', schedule); for (const box of [policyBox, sessionBox, strictBox, utcBox]) box.addEventListener('input', schedule); -$('#pg-load-policy').addEventListener('click', async () => { - policyBox.value = await (await fetch(new URL('./sqlquack.toml', import.meta.url))).text(); +$('#pg-load-policy').addEventListener('click', () => { + policyBox.value = repositoryPolicy; schedule(); }); +modeButton.addEventListener('click', () => { + usePolicy(policyMode() === 'repository' ? 'strict' : 'repository'); + run(); +}); rowsBox.addEventListener('input', () => latest && duckdb && schedule()); sqlBox.addEventListener('scroll', () => (marks.scrollTop = sqlBox.scrollTop)); $('#pg-example').addEventListener('change', (event) => { @@ -256,6 +284,8 @@ if (!params.get('sql')) showCaption(params.get('example')); schemaBox.value = SCHEMA; rowsBox.value = ROWS; setStage('parse', 'loading', '', 'loading the engine'); +repositoryPolicy = await (await fetch(new URL('./sqlquack.toml', import.meta.url))).text(); +usePolicy(params.get('policy') === 'strict' ? 'strict' : 'repository'); await engineReady; await run(); if (params.has('run')) execute(); diff --git a/website/public/styles.css b/website/public/styles.css index 3ed87ab92..b66ddcfde 100644 --- a/website/public/styles.css +++ b/website/public/styles.css @@ -1264,8 +1264,9 @@ footer a { .pg-stage header { display: flex; + flex-wrap: wrap; align-items: center; - gap: 10px; + gap: 6px 10px; padding: 12px 16px; } @@ -1295,6 +1296,21 @@ footer a { line-height: 1.5; } +/* The one-click policy switch in stage 3's header, beside the policy it names. */ +.pg-stage .pg-mode { + color: var(--muted-foreground); + font-size: 12px; + white-space: nowrap; +} + +#stage-translate .pg-sub { + white-space: nowrap; +} + +.pg-stage .pg-mode:hover { + color: var(--foreground); +} + .pg-status { padding: 2px 8px; border-radius: var(--radius); diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index 2d40f575b..bc16e23f9 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -90,7 +90,7 @@ test('the conversion budget is timed in the browser and refuses with SQ0006 when assert.match(report.translate.diagnostics[0].label, /beyond the budget of 0 ms/); }); -test('every example stops at the stage it is meant to, and the default translates cleanly', async () => { +test('every example stops at the stage it is meant to, under the default policy and strict', async () => { const { EXAMPLES, CAPTIONS, SCHEMA: PAGE_SCHEMA } = await import('../public/playground/examples.js'); const expected = { valid: ['pass', 'pass', 'pass'], @@ -101,12 +101,18 @@ test('every example stops at the stage it is meant to, and the default translate refused: ['pass', 'pass', 'fail'], }; assert.deepEqual(Object.keys(EXAMPLES).sort(), Object.keys(expected).sort()); - for (const [name, sql] of Object.entries(EXAMPLES)) { - const report = JSON.parse(analyze(sql, PAGE_SCHEMA)); - assert.deepEqual(['parse', 'checks', 'translate'].map((stage) => report[stage].status), expected[name], name); + // The page starts on the repository's policy under a UTC session; its switch turns to strict. + const policy = readFileSync(new URL('../public/playground/sqlquack.toml', import.meta.url), 'utf8'); + const modes = { repository: JSON.stringify({ policy, utc_session: true }), strict: undefined }; + for (const [mode, settings] of Object.entries(modes)) { + for (const [name, sql] of Object.entries(EXAMPLES)) { + const report = JSON.parse(analyze(sql, PAGE_SCHEMA, settings)); + const stages = ['parse', 'checks', 'translate'].map((stage) => report[stage].status); + assert.deepEqual(stages, expected[name], `${name} under ${mode}`); + } + // The one deliberate refusal is a known divergence, explained by its caption, under both. + const refused = JSON.parse(analyze(EXAMPLES.refused, PAGE_SCHEMA, settings)).translate; + assert.deepEqual(refused.diagnostics.map((d) => d.code), ['SQ0008'], mode); } - // The one deliberate refusal is a known divergence, explained by its caption. - const refused = JSON.parse(analyze(EXAMPLES.refused, PAGE_SCHEMA)).translate; - assert.deepEqual(refused.diagnostics.map((d) => d.code), ['SQ0008']); assert.ok(CAPTIONS.refused); }); From b9f0946252703260c2fba42873dc08862ae8a310 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:05:32 +0000 Subject: [PATCH 05/12] corpus: Snowflake compile receipts for 25 everyday queries of uncertain validity Each query ran once over the corpus schema as empty CTEs: EXPLAIN first, then the compiling ones executed for their result columns. --- corpora/everyday/snowflake-receipts.json | 411 +++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 corpora/everyday/snowflake-receipts.json diff --git a/corpora/everyday/snowflake-receipts.json b/corpora/everyday/snowflake-receipts.json new file mode 100644 index 000000000..8cf7da8c8 --- /dev/null +++ b/corpora/everyday/snowflake-receipts.json @@ -0,0 +1,411 @@ +{ + "schema": "corpora/everyday/schema.json @ wip/battle-corpus 6622ffec, as empty CTEs (WITH AS (SELECT CAST(NULL AS ) AS , ... WHERE FALSE)) before each query", + "screen": "EXPLAIN USING TEXT per query; compiling queries then executed on the empty inputs", + "results": [ + { + "query": 1, + "sql": "SELECT order_id, FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 2, + "sql": "SELECT order_id FROM orders WHERE status == 'shipped'", + "compiles": false, + "errno": 2139, + "sqlstate": "02000", + "message": "SQL compilation error: Unknown function \"==\"." + }, + { + "query": 3, + "sql": "SELECT order_id FROM orders;;", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 4, + "sql": "SELECT order_id FROM orders LIMIT", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 5, + "sql": "SELECT order_id FROM orders o JOIN customers c", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 6, + "sql": "SELECT order_id FROM orders WHERE status IN 'shipped'", + "compiles": false, + "errno": 1003, + "sqlstate": "42000", + "message": "SQL compilation error: syntax error line 1 at position 2,023 unexpected ''shipped''." + }, + { + "query": 7, + "sql": "SELECT order_id FROM orders WHERE total >> 10", + "compiles": false, + "errno": 1003, + "sqlstate": "42000", + "message": "SQL compilation error: syntax error line 1 at position 2,019 unexpected '>>'." + }, + { + "query": 8, + "sql": "SELECT * FROM orders TOP 5", + "compiles": false, + "errno": 1003, + "sqlstate": "42000", + "message": "SQL compilation error: syntax error line 1 at position 2,004 unexpected '5'." + }, + { + "query": 9, + "sql": "SELECT ARRAY_CONTAINS('urgent'::VARIANT, tags) AS urgent FROM tickets", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "URGENT", + "nullable": true, + "precision": null, + "scale": null, + "type_code": 13, + "type_name": "BOOLEAN" + } + ] + }, + { + "query": 10, + "sql": "SELECT COLLATE(email, 'en-ci') AS e FROM customers", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": 134217728, + "name": "E", + "nullable": true, + "precision": null, + "scale": null, + "type_code": 2, + "type_name": "TEXT" + } + ] + }, + { + "query": 11, + "sql": "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", + "compiles": false, + "errno": 1003, + "sqlstate": "42000", + "message": "SQL compilation error: syntax error line 1 at position 2,002 unexpected '('." + }, + { + "query": 12, + "sql": "SELECT DATE_PART('dow', ordered_at) AS dow FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "DOW", + "nullable": true, + "precision": 2, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 13, + "sql": "SELECT DATE_TRUNC(month, ordered_at) AS month FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "MONTH", + "nullable": true, + "precision": 0, + "scale": 9, + "type_code": 8, + "type_name": "TIMESTAMP_NTZ" + } + ] + }, + { + "query": 14, + "sql": "SELECT LENGTH(order_id, 2) AS x FROM orders", + "compiles": false, + "errno": 939, + "sqlstate": "22023", + "message": "SQL compilation error: error line 1 at position 1,986 too many arguments for function [LENGTH(ORDERS.ORDER_ID, 2)] expected 1, got 2" + }, + { + "query": 15, + "sql": "SELECT customer_id, email FROM customers UNION ALL BY NAME SELECT email, customer_id FROM customers", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "CUSTOMER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + }, + { + "internal_size": 134217728, + "name": "EMAIL", + "nullable": true, + "precision": null, + "scale": null, + "type_code": 2, + "type_name": "TEXT" + } + ] + }, + { + "query": 16, + "sql": "SELECT o.order_id FROM orders o ASOF JOIN payments p MATCH_CONDITION (o.ordered_at >= p.paid_at) ON o.order_id = p.order_id", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 17, + "sql": "SELECT order_id FROM orders FETCH FIRST 10 ROWS ONLY", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 18, + "sql": "SELECT order_id FROM orders WHERE (customer_id, status) IN (SELECT customer_id, 'shipped' FROM customers)", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 19, + "sql": "SELECT order_id FROM orders WHERE ordered_at > 5", + "compiles": false, + "errno": 1038, + "sqlstate": "22023", + "message": "SQL compilation error: Can not convert parameter '5' of type [NUMBER(1,0)] into expected type [TIMESTAMP_NTZ(9)]" + }, + { + "query": 20, + "sql": "SELECT order_id FROM orders o, orders o", + "compiles": false, + "errno": 2027, + "sqlstate": "42601", + "message": "SQL compilation error: duplicate alias 'O'" + }, + { + "query": 21, + "sql": "SELECT order_id id, total amount FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + }, + { + "internal_size": null, + "name": "AMOUNT", + "nullable": true, + "precision": 12, + "scale": 2, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 22, + "sql": "SELECT order_id, HASH_AGG(*) AS h FROM orders GROUP BY order_id", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + }, + { + "internal_size": null, + "name": "H", + "nullable": false, + "precision": 19, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 23, + "sql": "SELECT order_id, discount IS DISTINCT FROM 0 AS has_discount FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + }, + { + "internal_size": null, + "name": "HAS_DISCOUNT", + "nullable": true, + "precision": null, + "scale": null, + "type_code": 13, + "type_name": "BOOLEAN" + } + ] + }, + { + "query": 24, + "sql": "SELECT order_id, total + 'abc' AS x FROM orders", + "compiles": true, + "outcome": "rows", + "error": null, + "columns": [ + { + "internal_size": null, + "name": "ORDER_ID", + "nullable": true, + "precision": 38, + "scale": 0, + "type_code": 0, + "type_name": "FIXED" + }, + { + "internal_size": null, + "name": "X", + "nullable": true, + "precision": 19, + "scale": 5, + "type_code": 0, + "type_name": "FIXED" + } + ] + }, + { + "query": 25, + "sql": "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", + "compiles": false, + "errno": 904, + "sqlstate": "42000", + "message": "SQL compilation error: error line 1 at position 1,996 invalid identifier 'TOTAL'" + } + ] +} \ No newline at end of file From 40708120462984efaf183565b907733fd1a7c910 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:10:09 +0000 Subject: [PATCH 06/12] test(corpus): judge everyday queries' validity by Snowflake's receipts The 25 queries Snowflake compiled or rejected carry its verdict in a snowflake field, which decides whether they count as valid SQL in the translation rates: 287 of 436 strict and 356 of 436 repository. The test also lists the 8 queries Snowflake rejects that pass SQLQuack's parse and checks, so closing each gap updates the list. --- corpora/everyday/queries.toml | 47 +++++++++------- crates/sqlquack/src/analysis/tests/helpers.rs | 56 ++++++++++++++++++- .../analysis/tests/test_everyday_corpus.rs | 25 ++++++++- .../sqlquack/src/analysis/tests/test_types.rs | 4 ++ 4 files changed, 110 insertions(+), 22 deletions(-) diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 0e17c5e5f..7308e1fc7 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -3,7 +3,9 @@ # offers as "strict": "translated", "parse ", "check " or "refused ". # `expect_repository` is the outcome under the repository's sqlquack.toml with a UTC session, the # playground's default. The test fails when an outcome changes; update the expectation when a -# change is intended. `uncertain` marks a query whose Snowflake validity is not yet recorded. +# change is intended. `snowflake` is Snowflake's own verdict where one was recorded, "compiles" or +# "rejects ", from snowflake-receipts.json; it decides whether a query counts as valid SQL in +# the translation rates, over its category. [[query]] category = "literals" @@ -310,7 +312,7 @@ category = "select_star" sql = '''SELECT * FROM orders TOP 5''' expect = "parse SQ0001" expect_repository = "parse SQ0001" -uncertain = true +snowflake = "rejects 1003" [[query]] category = "select_star" @@ -371,7 +373,7 @@ category = "columns" sql = '''SELECT order_id id, total amount FROM orders''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "columns" @@ -552,7 +554,7 @@ category = "columns" sql = '''SELECT order_id FROM orders FETCH FIRST 10 ROWS ONLY''' expect = "refused SQ0002 node:select" expect_repository = "refused SQ0002 node:select" -uncertain = true +snowflake = "compiles" [[query]] category = "columns" @@ -751,7 +753,7 @@ category = "aggregates" sql = '''SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders''' expect = "refused SQ0002 fn:COUNT" expect_repository = "refused SQ0002 fn:COUNT" -uncertain = true +snowflake = "rejects 1003" [[query]] category = "aggregates" @@ -1058,7 +1060,7 @@ category = "joins" sql = '''SELECT o.order_id FROM orders o ASOF JOIN payments p MATCH_CONDITION (o.ordered_at >= p.paid_at) ON o.order_id = p.order_id''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "ctes" @@ -1323,7 +1325,7 @@ category = "dates" sql = '''SELECT DATE_TRUNC(month, ordered_at) AS month FROM orders''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "dates" @@ -1414,7 +1416,7 @@ category = "dates" sql = '''SELECT DATE_PART('dow', ordered_at) AS dow FROM orders''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "dates" @@ -1703,7 +1705,7 @@ category = "conditionals" sql = '''SELECT order_id, discount IS DISTINCT FROM 0 AS has_discount FROM orders''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "conditionals" @@ -1962,7 +1964,7 @@ category = "strings" sql = '''SELECT COLLATE(email, 'en-ci') AS e FROM customers''' expect = "refused SQ0002 fn:COLLATE" expect_repository = "refused SQ0002 fn:COLLATE" -uncertain = true +snowflake = "compiles" [[query]] category = "strings" @@ -2251,7 +2253,7 @@ category = "semi_structured" sql = '''SELECT ARRAY_CONTAINS('urgent'::VARIANT, tags) AS urgent FROM tickets''' expect = "refused SQ0008 node:column" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "semi_structured" @@ -2641,7 +2643,7 @@ category = "set_operations" sql = '''SELECT customer_id, email FROM customers UNION ALL BY NAME SELECT email, customer_id FROM customers''' expect = "translated" expect_repository = "translated" -uncertain = true +snowflake = "compiles" [[query]] category = "misc" @@ -2692,7 +2694,7 @@ category = "misc" sql = '''SELECT order_id FROM orders WHERE (customer_id, status) IN (SELECT customer_id, 'shipped' FROM customers)''' expect = "refused SQ0002 node:tuple" expect_repository = "refused SQ0002 node:tuple" -uncertain = true +snowflake = "compiles" [[query]] category = "misc" @@ -2759,7 +2761,7 @@ category = "misc" sql = '''SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p''' expect = "refused SQ0003 node:pivot" expect_repository = "refused SQ0003 node:pivot" -uncertain = true +snowflake = "rejects 904" [[query]] category = "misc" @@ -2772,7 +2774,7 @@ category = "misc" sql = '''SELECT order_id, HASH_AGG(*) AS h FROM orders GROUP BY order_id''' expect = "refused SQ0002 fn:HASH_AGG" expect_repository = "refused SQ0002 fn:HASH_AGG" -uncertain = true +snowflake = "compiles" [[query]] category = "misc" @@ -2905,6 +2907,7 @@ category = "invalid_syntax" sql = '''SELECT order_id, FROM orders''' expect = "translated" expect_repository = "translated" +snowflake = "compiles" [[query]] category = "invalid_syntax" @@ -2953,6 +2956,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders LIMIT''' expect = "refused SQ0003 node:limit" expect_repository = "refused SQ0003 node:limit" +snowflake = "compiles" [[query]] category = "invalid_syntax" @@ -2971,6 +2975,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders o JOIN customers c''' expect = "refused SQ0002 join:inner" expect_repository = "refused SQ0002 join:inner" +snowflake = "compiles" [[query]] category = "invalid_syntax" @@ -2983,6 +2988,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status == 'shipped'''' expect = "translated" expect_repository = "translated" +snowflake = "rejects 2139" [[query]] category = "invalid_syntax" @@ -3013,6 +3019,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status IN 'shipped'''' expect = "refused SQ0002 node:in" expect_repository = "refused SQ0002 node:in" +snowflake = "rejects 1003" [[query]] category = "invalid_syntax" @@ -3073,6 +3080,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders;;''' expect = "translated" expect_repository = "translated" +snowflake = "compiles" [[query]] category = "invalid_syntax" @@ -3085,6 +3093,7 @@ category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total >> 10''' expect = "refused SQ0002 node:bitwise_right_shift" expect_repository = "refused SQ0002 node:bitwise_right_shift" +snowflake = "rejects 1003" [[query]] category = "invalid_syntax" @@ -3211,7 +3220,7 @@ category = "invalid_semantics" sql = '''SELECT order_id, total + 'abc' AS x FROM orders''' expect = "refused SQ0003 node:add" expect_repository = "refused SQ0003 node:add" -uncertain = true +snowflake = "compiles" [[query]] category = "invalid_semantics" @@ -3224,14 +3233,14 @@ category = "invalid_semantics" sql = '''SELECT order_id FROM orders WHERE ordered_at > 5''' expect = "refused SQ0003 node:gt" expect_repository = "refused SQ0003 node:gt" -uncertain = true +snowflake = "rejects 1038" [[query]] category = "invalid_semantics" sql = '''SELECT LENGTH(order_id, 2) AS x FROM orders''' expect = "refused SQ0002 fn:LENGTH" expect_repository = "refused SQ0002 fn:LENGTH" -uncertain = true +snowflake = "rejects 939" [[query]] category = "invalid_semantics" @@ -3262,4 +3271,4 @@ category = "invalid_semantics" sql = '''SELECT order_id FROM orders o, orders o''' expect = "refused SQ0004 statement" expect_repository = "refused SQ0004 statement" -uncertain = true +snowflake = "rejects 2027" diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index 6f5b650fa..4959566e3 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -824,7 +824,12 @@ pub(crate) fn everyday_translation_rates() -> String { .query .iter() .filter(|query| !EVERYDAY_NOT_TRANSLATABLE.contains(&query.category.as_str())) - .filter(|query| !query.category.starts_with(EVERYDAY_INVALID_PREFIX)) + .filter(|query| { + query.snowflake.as_deref().map_or_else( + || !query.category.starts_with(EVERYDAY_INVALID_PREFIX), + |verdict| verdict == SNOWFLAKE_COMPILES, + ) + }) .collect(); let strict = valid .iter() @@ -841,6 +846,52 @@ pub(crate) fn everyday_translation_rates() -> String { ) } +/// Queries Snowflake rejects that pass SQLQuack's parse and checks, once every receipt matches. +pub(crate) fn everyday_snowflake_rejections_not_caught() -> Vec { + let (root, corpus) = everyday_corpus(); + let text = + std::fs::read_to_string(root.join("snowflake-receipts.json")).expect("receipts read"); + let receipts: serde_json::Value = serde_json::from_str(&text).expect("receipts parse"); + let results = receipts["results"] + .as_array() + .expect("receipts hold results"); + for receipt in results { + let sql = receipt["sql"] + .as_str() + .expect("a receipt names its SQL") + .trim(); + let verdict = receipt["compiles"] + .as_bool() + .filter(|compiles| *compiles) + .map_or_else( + || format!("rejects {}", receipt["errno"]), + |_| SNOWFLAKE_COMPILES.to_owned(), + ); + let query = corpus.query.iter().find(|query| query.sql.trim() == sql); + assert_eq!( + query.and_then(|query| query.snowflake.clone()), + Some(verdict), + "{sql}" + ); + } + let strict = everyday_arguments(&root, true); + corpus + .query + .iter() + .filter(|query| { + query + .snowflake + .as_deref() + .is_some_and(|verdict| verdict != SNOWFLAKE_COMPILES) + }) + .filter(|query| { + let outcome = everyday_outcome(&query.sql, &strict); + !outcome.starts_with("parse ") && !outcome.starts_with("check ") + }) + .map(|query| query.sql.clone()) + .collect() +} + /// The everyday corpus directory and its queries. fn everyday_corpus() -> (std::path::PathBuf, EverydayCorpus) { let root = repository_root().join(EVERYDAY_CORPUS); @@ -905,6 +956,9 @@ const EVERYDAY_NOT_TRANSLATABLE: [&str; 1] = ["ddl_dml"]; /// The prefix of corpus categories of SQL Snowflake rejects. const EVERYDAY_INVALID_PREFIX: &str = "invalid"; +/// Snowflake's verdict on a query it compiles. +const SNOWFLAKE_COMPILES: &str = "compiles"; + /// The outcome of a query that converts. const TRANSLATED: &str = "translated"; diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index 095538b08..b05d29a94 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -1,4 +1,7 @@ -use crate::analysis::tests::helpers::{everyday_outcome_mismatches, everyday_translation_rates}; +use crate::analysis::tests::helpers::{ + everyday_outcome_mismatches, everyday_snowflake_rejections_not_caught, + everyday_translation_rates, +}; use crate::analysis::tests::test_types::EverydayCorpusTestCase; #[test] @@ -6,7 +9,19 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i let test_cases = [EverydayCorpusTestCase { description: "every corpora/everyday query parses, checks, converts or refuses as recorded, under the strict built-in policy and the repository policy", expected_mismatches: Vec::new(), - expected_rates: "285 of 434 strict, 354 of 434 repository", + expected_rates: "287 of 436 strict, 356 of 436 repository", + // SQL Snowflake rejects (snowflake-receipts.json) that SQLQuack neither fails to parse nor + // flags in its checks; each should become a parse or check error. + expected_rejections_not_caught: vec![ + "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", + "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", + "SELECT order_id FROM orders WHERE status == 'shipped'", + "SELECT order_id FROM orders WHERE status IN 'shipped'", + "SELECT order_id FROM orders WHERE total >> 10", + "SELECT order_id FROM orders WHERE ordered_at > 5", + "SELECT LENGTH(order_id, 2) AS x FROM orders", + "SELECT order_id FROM orders o, orders o", + ], }]; for test_case in test_cases { @@ -22,5 +37,11 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i "{}", test_case.description ); + assert_eq!( + everyday_snowflake_rejections_not_caught(), + test_case.expected_rejections_not_caught, + "{}", + test_case.description + ); } } diff --git a/crates/sqlquack/src/analysis/tests/test_types.rs b/crates/sqlquack/src/analysis/tests/test_types.rs index 1696d38f9..89b58c10e 100644 --- a/crates/sqlquack/src/analysis/tests/test_types.rs +++ b/crates/sqlquack/src/analysis/tests/test_types.rs @@ -215,6 +215,7 @@ pub(crate) struct EverydayCorpusTestCase { pub(crate) description: &'static str, pub(crate) expected_mismatches: Vec, pub(crate) expected_rates: &'static str, + pub(crate) expected_rejections_not_caught: Vec<&'static str>, } /// The everyday corpus file. @@ -230,4 +231,7 @@ pub(crate) struct EverydayQuery { pub(crate) sql: String, pub(crate) expect: String, pub(crate) expect_repository: String, + /// Snowflake's recorded verdict, `compiles` or `rejects `, where one was recorded. + #[serde(default)] + pub(crate) snowflake: Option, } From 280f2e87d06440ee07d9b3709e884038812be80f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:39:52 +0000 Subject: [PATCH 07/12] Check unknown relations and functions, and word parse failures for readers (D150) - check takes --schema. With one, a relation the schema does not declare is SQ0004 at its name, and a call to a function that is neither a documented Snowflake function (from the syntax catalogue) nor a schema function is the new SQ0205 unknown_function. Without a schema, names stay unchecked. The playground's checks report the same as E200/E202 findings. - An unresolved double-quoted name gets help: use single quotes for a string literal: ''. - SQ0001 says what was expected and what was found in SQL's words, points an unclosed ( at where it opens, points a failure at the end of the input past the last token, and adds help to render dbt/Jinja templates when the SQL holds {{ }} or {% %}. The everyday corpus records the three outcomes this changes: FROM order and FROM orderz are now check SQ0004, NOSUCHFUNCTION is check SQ0205. --- Cargo.lock | 1 + corpora/everyday/queries.toml | 12 +- .../src/diagnostics/_helpers/mod.rs | 1 + .../src/diagnostics/_helpers/unparsed.rs | 111 ++ .../sqlquack-core/src/diagnostics/codes.lock | 1 + .../src/diagnostics/constants.rs | 60 + .../src/diagnostics/main/diagnose.rs | 33 +- .../src/diagnostics/tests/test_render.rs | 90 +- .../src/diagnostics/tests/test_types.rs | 8 + .../src/bindings/_helpers/check_json.rs | 2 +- crates/sqlquack-semantics/Cargo.toml | 3 + .../lineage/_helpers/checks/node_checks.rs | 29 +- .../src/lineage/_helpers/findings/checks.rs | 43 +- .../src/lineage/_helpers/findings/levels.rs | 14 +- .../src/lineage/_helpers/tracing/sources.rs | 4 +- .../src/lineage/constants.rs | 8 + .../sqlquack-semantics/src/lineage/models.rs | 54 +- .../src/lineage/snowflake_functions.txt | 1311 +++++++++++++++++ .../src/lineage/tests/helpers.rs | 111 +- .../lineage/tests/test_statement_findings.rs | 112 +- .../src/lineage/tests/test_types.rs | 11 + .../sqlquack-semantics/src/lineage/types.rs | 5 + .../src/parser/_helpers/types/data_types.rs | 17 +- crates/sqlquack/src/analysis/constants.rs | 5 + .../src/analysis/main/check_source.rs | 68 +- crates/sqlquack/src/analysis/tests/helpers.rs | 7 +- .../src/analysis/tests/test_locations.rs | 4 +- .../src/cli/_helpers/convert_command.rs | 25 +- crates/sqlquack/src/cli/models.rs | 3 + crates/sqlquack/tests/cli/helpers.rs | 6 + crates/sqlquack/tests/cli/test_check.rs | 34 +- crates/sqlquack/tests/cli/test_convert.rs | 29 +- website/public/playground/playground.js | 2 +- website/src/content/docs/docs/cli/check.mdx | 14 +- .../docs/docs/diagnostics/reference.mdx | 15 + website/tests/playground.test.mjs | 18 +- website/wasm/src/lib.rs | 56 +- 37 files changed, 2224 insertions(+), 103 deletions(-) create mode 100644 crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs create mode 100644 crates/sqlquack-semantics/src/lineage/snowflake_functions.txt diff --git a/Cargo.lock b/Cargo.lock index bb96d19d5..1a2796009 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1477,6 +1477,7 @@ dependencies = [ "sqlquack-core", "sqlquack-snowflake", "sqlquack-syntax", + "toml", ] [[package]] diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 7308e1fc7..572df7f7e 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -3134,14 +3134,14 @@ expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM order''' -expect = "refused SQ0004 statement" -expect_repository = "refused SQ0004 statement" +expect = "check SQ0004" +expect_repository = "check SQ0004" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orderz''' -expect = "refused SQ0004 statement" -expect_repository = "refused SQ0004 statement" +expect = "check SQ0004" +expect_repository = "check SQ0004" [[query]] category = "invalid_semantics" @@ -3245,8 +3245,8 @@ snowflake = "rejects 939" [[query]] category = "invalid_semantics" sql = '''SELECT NOSUCHFUNCTION(order_id) AS x FROM orders''' -expect = "refused SQ0002 fn:NOSUCHFUNCTION" -expect_repository = "refused SQ0002 fn:NOSUCHFUNCTION" +expect = "check SQ0205" +expect_repository = "check SQ0205" [[query]] category = "invalid_semantics" diff --git a/crates/sqlquack-core/src/diagnostics/_helpers/mod.rs b/crates/sqlquack-core/src/diagnostics/_helpers/mod.rs index b0e9a4eba..6cbeca015 100644 --- a/crates/sqlquack-core/src/diagnostics/_helpers/mod.rs +++ b/crates/sqlquack-core/src/diagnostics/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod code_entry; pub(crate) mod coded; pub(crate) mod line_index; +pub(crate) mod unparsed; diff --git a/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs b/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs new file mode 100644 index 000000000..45546e298 --- /dev/null +++ b/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs @@ -0,0 +1,111 @@ +//! A parse failure in SQL's words: what was expected, which bracket is open, and templates. + +use sqlquack_syntax::tokens::models::TokenType; +use sqlquack_syntax::{Dialect, LosslessToken, TokenKind, tokenize}; + +use crate::diagnostics::constants::{ + CLOSING_PAREN, END_OF_INPUT, EXPECTED_PREFIX, FOUND_SEPARATOR, NEAR_SUFFIX, TEMPLATE_HELP, + TEMPLATE_MARKERS, TOKEN_TEXTS, UNCLOSED_PAREN_HELP, UNEXPECTED_TOKEN, +}; +use crate::parsing::models::{SourceLines, SourceLocation}; + +/// What a parse failure's diagnostic says: its label, where it points and the likely fix. +pub(crate) struct Unparsed { + pub(crate) label: String, + pub(crate) location: Option, + pub(crate) help: Option, +} + +/// The parser's `message` at `location` in `source`, worded for the reader. +pub(crate) fn unparsed(message: &str, location: Option, source: &str) -> Unparsed { + let found = location.map(|at| found_text(source, at)); + let help = TEMPLATE_MARKERS + .iter() + .any(|marker| source.contains(marker)) + .then(|| TEMPLATE_HELP.to_owned()); + let Some((expected, got)) = message + .strip_prefix(EXPECTED_PREFIX) + .and_then(|rest| rest.split_once(FOUND_SEPARATOR)) + else { + let label = match (message.contains(UNEXPECTED_TOKEN), found) { + (true, Some(found)) if found.is_empty() => "unexpected end of the input".to_owned(), + (true, Some(found)) => format!("unexpected `{found}`"), + _ => message + .split(NEAR_SUFFIX) + .next() + .unwrap_or(message) + .to_owned(), + }; + return Unparsed { + label, + location, + help, + }; + }; + let expected = token_text(expected); + let at_end = got.starts_with(END_OF_INPUT) || found.as_ref().is_some_and(String::is_empty); + if expected == CLOSING_PAREN + && at_end + && let Some(open) = unclosed_paren(source) + { + return Unparsed { + label: "this `(` is never closed".to_owned(), + location: Some(SourceLines::new(source).location(open, open + 1)), + help: help.or_else(|| Some(UNCLOSED_PAREN_HELP.to_owned())), + }; + } + let found = match (at_end, found) { + (false, Some(found)) if !found.is_empty() => format!("`{found}`"), + _ => "the end of the input".to_owned(), + }; + Unparsed { + label: format!("expected {expected}, found {found}"), + location, + help, + } +} + +/// An expected token as SQL writes it: its text for a symbol or keyword, else the parser's words. +fn token_text(expected: &str) -> String { + if let Some((_, text)) = TOKEN_TEXTS.iter().find(|(name, _)| *name == expected) { + return format!("`{text}`"); + } + match expected.chars().all(char::is_alphanumeric) && expected.starts_with(char::is_uppercase) { + true => { + let mut keyword = String::new(); + for character in expected.chars() { + if character.is_uppercase() && !keyword.is_empty() { + keyword.push(' '); + } + keyword.push(character.to_ascii_uppercase()); + } + format!("`{keyword}`") + } + false => expected.to_owned(), + } +} + +/// The source's characters at `location`. +fn found_text(source: &str, location: SourceLocation) -> String { + source + .chars() + .skip(location.start) + .take(location.end.saturating_sub(location.start)) + .collect() +} + +/// The character offset of the last `(` the source never closes, outside strings and comments. +fn unclosed_paren(source: &str) -> Option { + let tokens: Vec = tokenize(source, Dialect::Snowflake).unwrap_or_default(); + let mut open: Vec = Vec::new(); + for token in tokens { + match token.kind { + TokenKind::Significant(TokenType::LParen) => open.push(token.start), + TokenKind::Significant(TokenType::RParen) => { + open.pop(); + } + _ => {} + } + } + open.last().map(|byte| source[..*byte].chars().count()) +} diff --git a/crates/sqlquack-core/src/diagnostics/codes.lock b/crates/sqlquack-core/src/diagnostics/codes.lock index 2d1443dbe..689b6e750 100644 --- a/crates/sqlquack-core/src/diagnostics/codes.lock +++ b/crates/sqlquack-core/src/diagnostics/codes.lock @@ -19,3 +19,4 @@ SQ0201 set_operation_all SQ0202 lock_wait SQ0203 qualify_without_window SQ0204 star_filter_drops_named_column +SQ0205 unknown_function diff --git a/crates/sqlquack-core/src/diagnostics/constants.rs b/crates/sqlquack-core/src/diagnostics/constants.rs index a6d54a88f..ce60392a0 100644 --- a/crates/sqlquack-core/src/diagnostics/constants.rs +++ b/crates/sqlquack-core/src/diagnostics/constants.rs @@ -24,6 +24,47 @@ pub const ZONED_LOCAL_TIME_HELP: &str = "convert the value to a wall time first: `CONVERT_TIMEZONE(, col)::TIMESTAMP_NTZ` for local day boundaries in a known zone"; /// The prefix of a complexity guard's message, before the rest of its `E_GUARD_*` name. pub const GUARD_PREFIX: &str = "E_GUARD_"; +/// How the parser's own messages start when it names what it expected, and what it found. +pub const EXPECTED_PREFIX: &str = "Expected "; +/// What separates the expected token from the found one in such a message. +pub const FOUND_SEPARATOR: &str = ", got "; +/// How the parser names the end of the input as the found token. +pub const END_OF_INPUT: &str = "end of input"; +/// Where the parser's message starts quoting the tokens around the failure. +pub const NEAR_SUFFIX: &str = " near ["; +/// What the parser's message says when it names only the token it did not expect. +pub const UNEXPECTED_TOKEN: &str = "Unexpected token"; +/// A closing parenthesis as an expected token is shown. +pub const CLOSING_PAREN: &str = "`)`"; +/// Each token name the parser expects, and its text in SQL. +pub const TOKEN_TEXTS: &[(&str, &str)] = &[ + ("LParen", "("), + ("RParen", ")"), + ("LBracket", "["), + ("RBracket", "]"), + ("LBrace", "{"), + ("RBrace", "}"), + ("Comma", ","), + ("Dot", "."), + ("Colon", ":"), + ("DColon", "::"), + ("Semicolon", ";"), + ("Eq", "="), + ("Star", "*"), + ("Arrow", "->"), +]; +/// The help for a `(` the SQL never closes. +pub const UNCLOSED_PAREN_HELP: &str = "add the missing `)` where the bracketed part ends"; +/// Text that marks a dbt or Jinja template, which is not SQL until rendered. +pub const TEMPLATE_MARKERS: [&str; 4] = ["{{", "}}", "{%", "%}"]; +/// The help for SQL that still holds template markers. +pub const TEMPLATE_HELP: &str = "render dbt/Jinja templates before checking"; +/// How an unresolved column's reason starts. +pub const UNRESOLVED_COLUMN_PREFIX: &str = "column `"; +/// The quote around a quoted identifier in Snowflake SQL. +pub const DOUBLE_QUOTE: char = '"'; +/// The help for a double-quoted name used as a value, before the name as a string literal. +pub const QUOTED_VALUE_HELP: &str = "use single quotes for a string literal:"; /// The longest source line shown in full; longer lines show a window around the span. pub const MAX_SHOWN_LINE: usize = 120; /// How many columns a tab takes when a source line is shown. @@ -35,6 +76,8 @@ pub const PARSE_FIDELITY: &str = "SQ0001"; pub const MISSING_RULE: &str = "SQ0002"; /// The construct's operand types fall outside every rule's proven conditions. pub const TYPE_CONDITION: &str = "SQ0003"; +/// The name of the code of a call to a function neither Snowflake nor the schema declares. +pub const UNKNOWN_FUNCTION: &str = "unknown_function"; /// A relation or column name does not resolve. pub const UNRESOLVED_NAME: &str = "SQ0004"; /// DuckDB cannot express the construct's meaning. @@ -284,4 +327,21 @@ pub const CODES: &[CodeEntry] = &[ case fails it whatever the table holds, so no schema is needed to see it.", example: r#"echo "SELECT * ILIKE '%_id' REPLACE (price * 2 AS price) FROM orders" | sqlquack check"#, }, + CodeEntry { + code: "SQ0205", + name: UNKNOWN_FUNCTION, + alias: None, + title: "Snowflake has no function of this name", + help: Some( + "check the function's spelling, or declare a user-defined function under `functions` \ + in the schema", + ), + explanation: "The call names a function that is neither one of Snowflake's documented \ + functions nor a function the schema declares, so Snowflake rejects the statement \ + (Snowflake error 2140, unknown function). SQLQuack checks calls only when a schema is \ + given, because a user-defined function can only be told apart from a misspelled one \ + by the schema's `functions`. Calls qualified with a database or schema are not \ + checked.", + example: r#"echo "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders" | sqlquack check --schema schema.json"#, + }, ]; diff --git a/crates/sqlquack-core/src/diagnostics/main/diagnose.rs b/crates/sqlquack-core/src/diagnostics/main/diagnose.rs index 5c494ea9c..f4f47ac04 100644 --- a/crates/sqlquack-core/src/diagnostics/main/diagnose.rs +++ b/crates/sqlquack-core/src/diagnostics/main/diagnose.rs @@ -3,10 +3,12 @@ use crate::diagnostics::_helpers::code_entry::code_entry; use crate::diagnostics::_helpers::coded::coded; use crate::diagnostics::_helpers::line_index::LineIndex; +use crate::diagnostics::_helpers::unparsed::unparsed; use crate::diagnostics::constants::{ - CASE_ONLY_MATCH, CASE_ONLY_MATCH_HELP, CONVERSION_BUDGET, DIVERGENCE_REFUSED, GUARD_PREFIX, - MISSING_RULE, NOT_EXPRESSIBLE, PARSE_FIDELITY, TYPE_CONDITION, UNPARSED_PREFIX, - UNRESOLVED_NAME, ZONED_LOCAL_TIME, ZONED_LOCAL_TIME_HELP, ZONED_LOCAL_TIME_NOTE, + CASE_ONLY_MATCH, CASE_ONLY_MATCH_HELP, CONVERSION_BUDGET, DIVERGENCE_REFUSED, DOUBLE_QUOTE, + GUARD_PREFIX, MISSING_RULE, NOT_EXPRESSIBLE, PARSE_FIDELITY, QUOTED_VALUE_HELP, TYPE_CONDITION, + UNPARSED_PREFIX, UNRESOLVED_COLUMN_PREFIX, UNRESOLVED_NAME, ZONED_LOCAL_TIME, + ZONED_LOCAL_TIME_HELP, ZONED_LOCAL_TIME_NOTE, }; use crate::diagnostics::models::Diagnostic; use crate::refusals::models::{Refusal, RefusalCategory}; @@ -19,11 +21,20 @@ pub fn diagnose(refusals: &[Refusal], source: &str) -> Vec { let (code, label) = code_and_label(refusal); let mut diagnostic = coded(code, label); diagnostic.span = refusal.location.map(|location| lines.span(location)); + if code == PARSE_FIDELITY { + let worded = unparsed(&diagnostic.label, refusal.location, source); + diagnostic.label = worded.label; + diagnostic.span = worded.location.map(|location| lines.span(location)); + diagnostic.help = worded.help; + } if refusal.category == RefusalCategory::UnresolvedName && refusal.reason.contains(CASE_ONLY_MATCH) { diagnostic.help = Some(CASE_ONLY_MATCH_HELP.to_owned()); } + if let Some(help) = quoted_value_help(refusal, source) { + diagnostic.help = Some(help); + } if refusal.category == RefusalCategory::TypeCondition && refusal.reason.contains(ZONED_LOCAL_TIME) { @@ -42,6 +53,22 @@ pub fn diagnose(refusals: &[Refusal], source: &str) -> Vec { diagnostics } +/// The fix for a double-quoted name that resolves to no column: a string literal was likely meant. +fn quoted_value_help(refusal: &Refusal, source: &str) -> Option { + let location = refusal.location?; + let written: String = source + .chars() + .skip(location.start) + .take(location.end.saturating_sub(location.start)) + .collect(); + let value = written + .strip_prefix(DOUBLE_QUOTE)? + .strip_suffix(DOUBLE_QUOTE)?; + (refusal.category == RefusalCategory::UnresolvedName + && refusal.reason.starts_with(UNRESOLVED_COLUMN_PREFIX)) + .then(|| format!("{QUOTED_VALUE_HELP} '{}'", value.replace('\'', "''"))) +} + /// The code a refusal maps to, and the label left once its code's own wording is removed. fn code_and_label(refusal: &Refusal) -> (&'static str, String) { let code = match refusal.category { diff --git a/crates/sqlquack-core/src/diagnostics/tests/test_render.rs b/crates/sqlquack-core/src/diagnostics/tests/test_render.rs index 6da1f8c50..4f622b7a2 100644 --- a/crates/sqlquack-core/src/diagnostics/tests/test_render.rs +++ b/crates/sqlquack-core/src/diagnostics/tests/test_render.rs @@ -1,6 +1,6 @@ use crate::diagnostics::main::diagnose::diagnose; use crate::diagnostics::tests::helpers::{placed, rendered, unlocated, unparsed}; -use crate::diagnostics::tests::test_types::{GuardTestCase, RenderTestCase}; +use crate::diagnostics::tests::test_types::{GuardTestCase, RenderTestCase, UnparsedTestCase}; use crate::refusals::models::RefusalCategory; #[test] @@ -55,11 +55,12 @@ fn given_refusals_when_rendered_then_the_source_line_is_underlined_at_the_span() .to_owned(), }, RenderTestCase { - description: "a parse failure points at where the parser stopped, with its message as the label", + description: "an unclosed parenthesis is pointed at where it opens, with the fix as help", source: "SELECT (1".to_owned(), refusal: unparsed("SELECT (1"), - expected_text: "error[SQ0001]: the SQL does not parse\n --> query.sql:1:9\n |\n1 | \ - SELECT (1\n | ^ Expected RParen, got end of input ('') near [SELECT (1]\n" + expected_text: "error[SQ0001]: the SQL does not parse\n --> query.sql:1:8\n |\n1 | \ + SELECT (1\n | ^ this `(` is never closed\n |\n = help: add the missing `)` \ + where the bracketed part ends\n" .to_owned(), }, RenderTestCase { @@ -112,7 +113,7 @@ fn given_guard_failures_when_diagnosed_then_each_guard_has_its_own_code_and_form description: "50,000 added terms are one flat chain, not 50,000 levels of depth", source: format!("SELECT {} FROM", vec!["1"; 50_000].join(" + ")), expected_code: ("SQ0001", None), - expected_label: "Expected table name or subquery, got From", + expected_label: "expected table name or subquery, found the end of the input", }, ]; for test_case in test_cases { @@ -128,3 +129,82 @@ fn given_guard_failures_when_diagnosed_then_each_guard_has_its_own_code_and_form ); } } + +#[test] +fn given_everyday_parse_failures_when_diagnosed_then_the_label_names_what_was_expected() { + let test_cases = [ + UnparsedTestCase { + description: "a CASE without END names the missing keyword and what came instead", + source: "SELECT CASE WHEN total > 1 THEN 'a' FROM orders", + expected_label: "expected `END`, found `FROM`", + expected_column: 37, + expected_help: None, + }, + UnparsedTestCase { + description: "an unclosed parenthesis is pointed at where it opens", + source: "SELECT order_id FROM orders WHERE (total > 1", + expected_label: "this `(` is never closed", + expected_column: 35, + expected_help: Some("add the missing `)` where the bracketed part ends"), + }, + UnparsedTestCase { + description: "a parenthesis inside a string is no bracket", + source: "SELECT '(' AS a, (1", + expected_label: "this `(` is never closed", + expected_column: 18, + expected_help: Some("add the missing `)` where the bracketed part ends"), + }, + UnparsedTestCase { + description: "a clause cut off at the end points past its last word", + source: "SELECT a FROM orders ORDER BY", + expected_label: "unexpected end of the input", + expected_column: 30, + expected_help: None, + }, + UnparsedTestCase { + description: "a missing FROM item says the input ended", + source: "SELECT a FROM", + expected_label: "expected table name or subquery, found the end of the input", + expected_column: 14, + expected_help: None, + }, + UnparsedTestCase { + description: "an unclosed call before FROM names both tokens", + source: "SELECT (a + 1 FROM orders", + expected_label: "expected `)`, found `FROM`", + expected_column: 15, + expected_help: None, + }, + UnparsedTestCase { + description: "a doubled operator names the operator, not its token kind", + source: "SELECT a FROM t WHERE a = = 1", + expected_label: "unexpected `=`", + expected_column: 27, + expected_help: None, + }, + UnparsedTestCase { + description: "a dbt model is rendered before it is checked", + source: "select\n {{ dbt_utils.generate_surrogate_key(['order_id', 'line_number']) }} as order_item_key,\n order_id\nfrom {{ ref('stg_order_items') }}", + expected_label: "expected `:`, found `}`", + expected_column: 70, + expected_help: Some("render dbt/Jinja templates before checking"), + }, + ]; + for test_case in test_cases { + let diagnostics = diagnose(&[unparsed(test_case.source)], test_case.source); + assert_eq!( + ( + diagnostics[0].label.as_str(), + diagnostics[0].span.map(|span| span.column), + diagnostics[0].help.as_deref() + ), + ( + test_case.expected_label, + Some(test_case.expected_column), + test_case.expected_help + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/diagnostics/tests/test_types.rs b/crates/sqlquack-core/src/diagnostics/tests/test_types.rs index 9a6bfd406..cb1667769 100644 --- a/crates/sqlquack-core/src/diagnostics/tests/test_types.rs +++ b/crates/sqlquack-core/src/diagnostics/tests/test_types.rs @@ -26,3 +26,11 @@ pub(crate) struct GuardNameTestCase { pub(crate) roots: &'static [&'static str], pub(crate) expected_unregistered: Vec, } + +pub(crate) struct UnparsedTestCase { + pub(crate) description: &'static str, + pub(crate) source: &'static str, + pub(crate) expected_label: &'static str, + pub(crate) expected_column: usize, + pub(crate) expected_help: Option<&'static str>, +} diff --git a/crates/sqlquack-python/src/bindings/_helpers/check_json.rs b/crates/sqlquack-python/src/bindings/_helpers/check_json.rs index 4171d3abc..11091a814 100644 --- a/crates/sqlquack-python/src/bindings/_helpers/check_json.rs +++ b/crates/sqlquack-python/src/bindings/_helpers/check_json.rs @@ -11,7 +11,7 @@ use crate::bindings::models::Rendered; /// The check report of `sql` as JSON, naming the source `path`; `ValueError` without SQL. #[pyfunction] pub(crate) fn check_json(sql: &str, path: &str) -> PyResult { - let diagnostics = check_source(sql).map_err(PyValueError::new_err)?; + let diagnostics = check_source(sql, None).map_err(PyValueError::new_err)?; let rendered = render_diagnostics(&diagnostics, sql, path); let report = CheckReport { ok: diagnostics.is_empty(), diff --git a/crates/sqlquack-semantics/Cargo.toml b/crates/sqlquack-semantics/Cargo.toml index ccb580b66..93a67628b 100644 --- a/crates/sqlquack-semantics/Cargo.toml +++ b/crates/sqlquack-semantics/Cargo.toml @@ -16,5 +16,8 @@ sqlquack-syntax.workspace = true sqlquack-core.workspace = true sqlquack-snowflake.workspace = true +[dev-dependencies] +toml.workspace = true + [lints] workspace = true diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs index bdcd9bd08..cf37cc935 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/node_checks.rs @@ -4,6 +4,7 @@ use sqlquack_syntax::Expression; use sqlquack_syntax::expressions::names::models::Literal; use sqlquack_syntax::expressions::queries::models::Select; use sqlquack_syntax::expressions::types::models::DataType; +use sqlquack_syntax::tokens::models::Span; use crate::lineage::_helpers::checks::coercion::{ Engine, arithmetic, comparable, invalid_literal_for, invalid_temporal_literal, literal_coerces, @@ -19,7 +20,10 @@ use crate::lineage::_helpers::checks::relational::{ use crate::lineage::_helpers::checks::signatures::{ argument_finding, check_call, dispatch_name, has_signature, located, unify, }; -use crate::lineage::constants::{DUCKDB_KEYWORD_ONLY_FUNCTIONS, NAME_PHASE, TYPE_PHASE}; +use crate::lineage::constants::{ + DUCKDB_KEYWORD_ONLY_FUNCTIONS, NAME_PHASE, NAME_SEPARATOR, SNOWFLAKE_FUNCTION_NAMES, + TYPE_PHASE, UNKNOWN_FUNCTION_CODE, +}; use crate::lineage::models::{LineageContext, LineageScope, NameFinding}; /// COALESCE without arguments, which polyglot checks against the catalog under this name. @@ -57,6 +61,29 @@ pub(crate) fn node_findings(node: &Expression, level: NodeLevel<'_, '_>) -> Vec< { return findings; } + if let Expression::Function(function) = node + && level.engine() == Engine::Snowflake + && level.context.schema.closed() + && !function.name.contains(NAME_SEPARATOR) + && !function.name.contains(char::is_whitespace) + && !SNOWFLAKE_FUNCTION_NAMES + .lines() + .any(|known| known.eq_ignore_ascii_case(&function.name)) + { + findings.push(NameFinding { + written: function.name.clone(), + ..issue( + (UNKNOWN_FUNCTION_CODE, NAME_PHASE), + format!("Unknown function '{}' for dialect Snowflake", function.name), + function.span.0.map(|span| Span { + end: span.start + function.name.chars().count(), + ..span + }), + node, + ) + }); + return findings; + } let family = |expression: &Expression| level.family(expression); let mut own: Vec = match node { Expression::Function(function) => { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs b/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs index b9be59cd3..03375d149 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs @@ -7,11 +7,12 @@ use sqlquack_syntax::expressions::queries::models::Select; use crate::lineage::_helpers::checks::scope_rules::output_width; use crate::lineage::_helpers::tracing::queries::empty_scope; +use crate::lineage::_helpers::tracing::queries::unannotated; use crate::lineage::_helpers::tracing::resolution::key; -use crate::lineage::_helpers::tracing::sources::item_sources; +use crate::lineage::_helpers::tracing::sources::{item_sources, relation_name, written_name}; use crate::lineage::constants::{ ALIAS_COUNT_CODE, AMBIGUOUS_COLUMN_CODE, CONNECT_BY_LEVEL, NAME_PHASE, UNKNOWN_COLUMN_CODE, - UNKNOWN_QUALIFIER_CODE, + UNKNOWN_QUALIFIER_CODE, UNKNOWN_TABLE_CODE, }; use crate::lineage::constants::{ DUCKDB_PROFILE, DUCKDB_ROWID, METADATA_PREFIX, POSITIONAL_PREFIX, SEQUENCE_COLUMNS, @@ -232,6 +233,44 @@ fn lacks(side: &[ScopeSource], wanted: &str, context: LineageContext<'_>) -> boo !side.iter().any(unjudged) && !side.iter().any(|source| offers(source, wanted, context)) } +/// A FROM item reading a stored relation a closed schema does not declare, as polyglot reports it. +pub(crate) fn unknown_relation_findings( + item: &Expression, + ctes: &[NamedQuery], + context: LineageContext<'_>, +) -> Vec { + let table = match unannotated(item) { + Expression::Table(table) => table, + Expression::Alias(alias) => match unannotated(&alias.this) { + Expression::Table(table) => table, + _ => return Vec::new(), + }, + _ => return Vec::new(), + }; + let name: String = key(&table.name, context); + let cte: bool = table.schema.is_none() + && table.catalog.is_none() + && ctes + .iter() + .any(|cte| context.profile.same_identifier(&cte.key, &name)); + let relation: String = relation_name(table, context); + if !context.schema.closed() + || cte + || context.schema.columns(&relation).is_some() + || context.schema.open_columns(&relation).is_some() + { + return Vec::new(); + } + vec![NameFinding { + code: UNKNOWN_TABLE_CODE, + message: format!("Unknown table '{}'", written_name(table)), + span: table.name.span, + written: table.name.name.clone(), + phase: NAME_PHASE, + ..NameFinding::default() + }] +} + /// UNPIVOT operands its closed input does not offer. pub(crate) fn unpivot_findings( item: &Expression, diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs index 91442cf4f..cb3e75475 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs @@ -11,13 +11,15 @@ use crate::lineage::_helpers::checks::node_checks::{NodeLevel, node_findings}; use crate::lineage::_helpers::checks::scope_rules::{RuleLevel, select_rules, set_order}; use crate::lineage::_helpers::checks::sets::{Branches, set_findings}; use crate::lineage::_helpers::findings::checks::{ - Level, alias_count_findings, column_finding, unpivot_findings, using_findings, + Level, alias_count_findings, column_finding, unknown_relation_findings, unpivot_findings, + using_findings, }; use crate::lineage::_helpers::tracing::expressions::date_part_argument; use crate::lineage::_helpers::tracing::queries::{ level_scope, output_key, unannotated, with_ctes_visiting, }; use crate::lineage::_helpers::tracing::resolution::key; +use crate::lineage::constants::UNKNOWN_TABLE_CODE; use crate::lineage::models::NameFinding; use crate::lineage::models::{LineageContext, LineageScope, NamedQuery}; @@ -68,7 +70,14 @@ fn visited_ctes<'a>( ) -> (Cow<'a, [NamedQuery]>, Vec) { let mut found: Vec = Vec::new(); let reach = with_ctes_visiting(with, (ctes, outer), context, &mut |cte, reach| { - found.extend(query_findings(&cte.this, (reach, outer, aliases), context)); + found.extend( + query_findings(&cte.this, (reach, outer, aliases), context) + .into_iter() + .filter(|finding| { + finding.code != UNKNOWN_TABLE_CODE + || !finding.written.eq_ignore_ascii_case(&cte.alias.name) + }), + ); }); (reach, found) } @@ -103,6 +112,7 @@ fn select_findings( (&ctes, reads, aliases), context, )); + own.extend(unknown_relation_findings(item, &ctes, context)); own.extend(unpivot_findings(item, &ctes, context)); } let mut outputs: Vec = aliases.to_vec(); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index f2f4980d5..d4d218a4d 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -361,7 +361,7 @@ fn table_source( } /// The relation's name as written, its parts joined by dots, in lower case. -fn written_name(table: &TableRef) -> String { +pub(crate) fn written_name(table: &TableRef) -> String { [ table.catalog.as_ref(), table.schema.as_ref(), @@ -375,7 +375,7 @@ fn written_name(table: &TableRef) -> String { } /// The relation's name, its parts resolved and joined by dots. -fn relation_name(table: &TableRef, context: LineageContext<'_>) -> String { +pub(crate) fn relation_name(table: &TableRef, context: LineageContext<'_>) -> String { [ table.catalog.as_ref(), table.schema.as_ref(), diff --git a/crates/sqlquack-semantics/src/lineage/constants.rs b/crates/sqlquack-semantics/src/lineage/constants.rs index c3359239c..14c9c5a8c 100644 --- a/crates/sqlquack-semantics/src/lineage/constants.rs +++ b/crates/sqlquack-semantics/src/lineage/constants.rs @@ -310,6 +310,14 @@ pub(crate) const DATE_PART_FUNCTIONS: [(&str, usize); 9] = [ ("DATE_TRUNC", 0), ("LAST_DAY", 1), ]; +/// The code of a stored relation a closed schema does not declare. +pub(crate) const UNKNOWN_TABLE_CODE: &str = "E200"; +/// The code of a call to a function neither Snowflake nor a closed schema declares. +pub(crate) const UNKNOWN_FUNCTION_CODE: &str = "E202"; +/// What separates the parts of a qualified name. +pub(crate) const NAME_SEPARATOR: char = '.'; +/// Snowflake's documented function names, one per line and sorted, from the syntax catalogue. +pub(crate) const SNOWFLAKE_FUNCTION_NAMES: &str = include_str!("snowflake_functions.txt"); /// The code of a column no FROM item offers. pub(crate) const UNKNOWN_COLUMN_CODE: &str = "E201"; /// The code of a FROM item with more column aliases than columns. diff --git a/crates/sqlquack-semantics/src/lineage/models.rs b/crates/sqlquack-semantics/src/lineage/models.rs index 932d01e8a..71a50f30e 100644 --- a/crates/sqlquack-semantics/src/lineage/models.rs +++ b/crates/sqlquack-semantics/src/lineage/models.rs @@ -1,6 +1,6 @@ //! Column lineage of a query: each output column, the source columns it reads, and how. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use sqlquack_syntax::expressions::names::models::Column; @@ -8,6 +8,7 @@ use sqlquack_syntax::tokens::models::Span; use crate::dialects::types::DialectProfile; use crate::lineage::types::{LineageSchema, SharedColumns}; +use crate::types::models::{ColumnSpec, Schema}; /// How an output column is derived from its source columns. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] @@ -226,3 +227,54 @@ pub struct NameFinding { /// The node the finding is about, as an address: findings of a phase follow tree order. pub(crate) node: usize, } + +/// A schema file's relations and functions, closed: a name it does not declare is a finding. +#[derive(Debug, Default)] +pub struct SchemaCatalog { + relations: BTreeMap>, + functions: BTreeSet, +} + +impl SchemaCatalog { + /// The catalog of `schema`, its columns typed as the schema declares them. + pub fn new(schema: &Schema) -> Self { + let relations: BTreeMap> = schema + .relations + .iter() + .map(|(relation, columns)| (relation.clone(), Self::named_columns(columns))) + .collect(); + Self { + relations, + functions: schema.functions.keys().cloned().collect(), + } + } + + /// A relation's declared columns as the name checks read them. + fn named_columns(columns: &[ColumnSpec]) -> Vec { + columns + .iter() + .map(|column| SchemaColumn { + name: column.name.clone(), + quoted: false, + data_type: Some(column.data_type.to_string()), + nullability: Nullability::Unknown, + }) + .collect() + } +} + +impl LineageSchema for SchemaCatalog { + fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { + self.relations.get(relation).map(Vec::as_slice) + } + + fn known_function(&self, name: &str) -> bool { + self.functions + .iter() + .any(|function| function.eq_ignore_ascii_case(name)) + } + + fn closed(&self) -> bool { + true + } +} diff --git a/crates/sqlquack-semantics/src/lineage/snowflake_functions.txt b/crates/sqlquack-semantics/src/lineage/snowflake_functions.txt new file mode 100644 index 000000000..02670d7fa --- /dev/null +++ b/crates/sqlquack-semantics/src/lineage/snowflake_functions.txt @@ -0,0 +1,1311 @@ +ABS +ACCUMULATE +ACCURACY +ACOS +ACOSH +ADD_MONTHS +ADD_ROW_ACCESS_POLICY_ON_EVENTS_VIEW +AGENT_RUN +AGG +AI_AGG +AI_CLASSIFY +AI_COMPLETE +AI_COUNT_TOKENS +AI_EMBED +AI_EXTRACT +AI_FILTER +AI_GENERATE_TABLE_DESC +AI_MULTI_EMBED +AI_PARSE_DOCUMENT +AI_REDACT +AI_SENTIMENT +AI_SIMILARITY +AI_SUMMARIZE +AI_SUMMARIZE_AGG +AI_TRANSCRIBE +AI_TRANSLATE +ALERT_HISTORY +ALL +ALL_SPACE_COUNT +ALL_SPACE_PERCENT +ALL_USER_NAMES +ANY +ANY_VALUE +APPLICATION_CALLBACK_HISTORY +APPLICATION_CONFIGURATION_VALUE_HISTORY +APPLICATION_JSON +APPLICATION_SPECIFICATION_STATUS_HISTORY +APPROXIMATE_JACCARD_INDEX +APPROXIMATE_SIMILARITY +APPROX_COUNT_DISTINCT +APPROX_PERCENTILE +APPROX_PERCENTILE_ACCUMULATE +APPROX_PERCENTILE_COMBINE +APPROX_PERCENTILE_ESTIMATE +APPROX_QUANTILE_20 +APPROX_QUANTILE_25 +APPROX_QUANTILE_40 +APPROX_QUANTILE_50 +APPROX_QUANTILE_60 +APPROX_QUANTILE_80 +APPROX_QUANTILE_95 +APPROX_QUANTILE_99 +APPROX_TOP_K +APPROX_TOP_K_ACCUMULATE +APPROX_TOP_K_COMBINE +APPROX_TOP_K_ESTIMATE +ARRAY +ARRAYS_OVERLAP +ARRAYS_TO_OBJECT +ARRAYS_ZIP +ARRAY_AGG +ARRAY_APPEND +ARRAY_CAT +ARRAY_COMPACT +ARRAY_CONSTRUCT +ARRAY_CONSTRUCT_COMPACT +ARRAY_CONSTRUCT_STRUCTURED +ARRAY_CONTAINS +ARRAY_DISTINCT +ARRAY_EXCEPT +ARRAY_FLATTEN +ARRAY_GENERATE_RANGE +ARRAY_INSERT +ARRAY_INTERSECTION +ARRAY_MAX +ARRAY_MIN +ARRAY_POSITION +ARRAY_PREPEND +ARRAY_REMOVE +ARRAY_REMOVE_AT +ARRAY_REPEAT +ARRAY_REVERSE +ARRAY_SIZE +ARRAY_SLICE +ARRAY_SORT +ARRAY_SUM +ARRAY_TO_STRING +ARRAY_UNION_AGG +ARRAY_UNIQUE_AGG +AS +ASCII +ASIN +ASINH +ASSOCIATE_SEMANTIC_CATEGORY_TAGS +AS_ARRAY +AS_BINARY +AS_BOOLEAN +AS_CHAR +AS_DATE +AS_DECIMAL +AS_DOUBLE +AS_INTEGER +AS_NUMBER +AS_OBJECT +AS_REAL +AS_TIME +AS_TIMESTAMP_LTZ +AS_TIMESTAMP_NTZ +AS_TIMESTAMP_TZ +AS_VARCHAR +AT +ATAN +ATAN2 +ATANH +AUTOMATIC_CLUSTERING_HISTORY +AUTO_REFRESH_REGISTRATION_HISTORY +AVAILABLE_LISTINGS +AVAILABLE_LISTING_REFRESH_HISTORY +AVG +BASE64_DECODE_BINARY +BASE64_DECODE_STRING +BASE64_ENCODE +BEFORE +BIND_VALUES +BITAND +BITAND_AGG +BITMAP_ABSOLUTE_POSITION +BITMAP_AND +BITMAP_AND_AGG +BITMAP_BIT_POSITION +BITMAP_BUCKET_NUMBER +BITMAP_CONSTRUCT_AGG +BITMAP_COUNT +BITMAP_OR +BITMAP_OR_AGG +BITMAP_TO_ARRAY +BITNOT +BITOR +BITOR_AGG +BITSHIFTLEFT +BITSHIFTRIGHT +BITXOR +BITXOR_AGG +BIT_LENGTH +BLANK_COUNT +BLANK_PERCENT +BOOLAND +BOOLAND_AGG +BOOLNOT +BOOLOR +BOOLOR_AGG +BOOLXOR +BOOLXOR_AGG +BUCKET_START +BUILD_SCOPED_FILE_URL +BUILD_STAGE_FILE_URL +CASE_FORMAT_VIOLATION_COUNT +CASE_FORMAT_VIOLATION_PERCENT +CAST +CBRT +CEIL +CHANGES +CHARINDEX +CHECK_JSON +CHECK_XML +CHR +CLASSIFY_TEXT +COALESCE +COLLATE +COLLATION +COMPLETE +COMPLETE_TASK_GRAPHS +COMPRESS +CONCAT +CONCAT_WS +CONDITIONAL_CHANGE_EVENT +CONDITIONAL_TRUE_EVENT +CONTAINS +CONTAINS_SPACES_COUNT +CONTAINS_SPACES_PERCENT +CONVERT_TIMEZONE +COPY_HISTORY +CORR +CORTEX_SEARCH_BATCH +CORTEX_SEARCH_DATA_SCAN +CORTEX_SEARCH_REFRESH_HISTORY +COS +COSH +COT +COUNT +COUNT_IF +COUNT_TOKENS +COVAR_POP +COVAR_SAMP +CUBE +CUME_DIST +CUMULATIVE_PRIVACY_LOSSES +CURRENT_ACCOUNT +CURRENT_ACCOUNT_NAME +CURRENT_AVAILABLE_ROLES +CURRENT_CLIENT +CURRENT_DATABASE +CURRENT_DATE +CURRENT_IP_ADDRESS +CURRENT_ORGANIZATION_NAME +CURRENT_ORGANIZATION_USER +CURRENT_REGION +CURRENT_ROLE +CURRENT_ROLE_TYPE +CURRENT_SCHEMA +CURRENT_SCHEMAS +CURRENT_SECONDARY_ROLES +CURRENT_SESSION +CURRENT_STATEMENT +CURRENT_TASK_GRAPHS +CURRENT_TIME +CURRENT_TIMESTAMP +CURRENT_TRANSACTION +CURRENT_USER +CURRENT_VERSION +CURRENT_WAREHOUSE +DATABASE_REFRESH_HISTORY +DATABASE_REFRESH_PROGRESS +DATABASE_REFRESH_PROGRESS_BY_JOB +DATABASE_REPLICATION_USAGE_HISTORY +DATABASE_STORAGE_USAGE_HISTORY +DATASKETCHES_HLL +DATASKETCHES_HLL_ACCUMULATE +DATASKETCHES_HLL_COMBINE +DATASKETCHES_HLL_ESTIMATE +DATA_AGENT_RUN +DATA_METRIC_FUNCTION_EXPECTATIONS +DATA_METRIC_FUNCTION_REFERENCES +DATA_METRIC_SCHEDULED_TIME +DATA_QUALITY_MONITORING_EXPECTATION_STATUS +DATA_QUALITY_MONITORING_RESULTS +DATA_TRANSFER_HISTORY +DATE +DATEADD +DATEDIFF +DATE_FROM_PARTS +DATE_PART +DATE_TRUNC +DAY +DAYNAME +DAYOFMONTH +DAYOFWEEK +DAYOFWEEKISO +DAYOFYEAR +DBT_PROJECT_EXECUTION_HISTORY +DCM_DEPLOYMENT_HISTORY +DECODE +DECOMPRESS_BINARY +DECOMPRESS_STRING +DECRYPT +DECRYPT_RAW +DEGREES +DENSE_RANK +DEREGISTER_EXTENSION +DIV0 +DIV0NULL +DP_INTERVAL_HIGH +DP_INTERVAL_LOW +DROP_ROW_ACCESS_POLICY_ON_EVENTS_VIEW +DUPLICATE_COUNT +DUPLICATE_PERCENT +DYNAMIC_TABLES +DYNAMIC_TABLE_GRAPH_HISTORY +DYNAMIC_TABLE_REFRESH_HISTORY +EDITDISTANCE +EMAIL_INTEGRATION_CONFIG +EMBED_TEXT_1024 +EMBED_TEXT_768 +EMPTY_STRING_COUNT +EMPTY_STRING_PERCENT +ENCRYPT +ENCRYPT_RAW +END +ENDSWITH +ENTITY_SENTIMENT +EQUAL_NULL +ESTIMATE_HYBRID_TABLE_STORAGE_USAGE +ESTIMATE_REMAINING_DP_AGGREGATES +EVALUATE_CANDIDATE_NETWORK_POLICY +EXECUTE_AI_EVALUATION +EXISTS +EXP +EXPLAIN_GRANTABLE_PRIVILEGES +EXPLAIN_JSON +EXPLAIN_PRIVILEGES +EXTERNAL_FUNCTIONS_HISTORY +EXTERNAL_TABLE_FILES +EXTERNAL_TABLE_FILE_REGISTRATION_HISTORY +EXTRACT +EXTRACT_ANSWER +EXTRACT_SEMANTIC_CATEGORIES +EXTREME_OUTLIER_COUNT +EXTREME_OUTLIER_IQR_COUNT +EXTREME_OUTLIER_IQR_PERCENT +EXTREME_OUTLIER_PERCENT +EXTREME_OUTLIER_ZSCORE_COUNT +EXTREME_OUTLIER_ZSCORE_PERCENT +FACTORIAL +FALSE_COUNT +FALSE_PERCENT +FILTER +FINETUNE +FIRST_VALUE +FLATTEN +FLOOR +FL_GET_CONTENT_TYPE +FL_GET_ETAG +FL_GET_FILE_TYPE +FL_GET_LAST_MODIFIED +FL_GET_RELATIVE_PATH +FL_GET_SCOPED_FILE_URL +FL_GET_SIZE +FL_GET_STAGE +FL_GET_STAGE_FILE_URL +FL_IS_AUDIO +FL_IS_COMPRESSED +FL_IS_DOCUMENT +FL_IS_IMAGE +FL_IS_VIDEO +FORMAT_STRING_TEMPLATE +FRESHNESS +FROM +FUTURE_TIMESTAMP_COUNT +FUTURE_TIMESTAMP_PERCENT +GENERATE_COLUMN_DESCRIPTION +GENERATE_POSTGRES_ACCESS_TOKEN_FOR_USER +GENERATE_SYNTHETIC_DATA +GENERATOR +GEOMETRIC_MEAN +GET +GETBIT +GETDATE +GETVARIABLE +GET_ABSOLUTE_PATH +GET_AI_EVALUATION_DATA +GET_AI_OBSERVABILITY_EVENTS +GET_AI_OBSERVABILITY_LOGS +GET_AI_RECORD_TRACE +GET_ANACONDA_PACKAGES_REPODATA +GET_AUTO_ENABLEMENT_STATUS +GET_CONDITION_QUERY_UUID +GET_CONTACTS +GET_DDL +GET_IGNORE_CASE +GET_JOB_HISTORY +GET_LINEAGE +GET_OBJECT_REFERENCES +GET_PATH +GET_PRESIGNED_URL +GET_PYTHON_PROFILER_OUTPUT +GET_QUERY_OPERATOR_STATS +GET_RELATIVE_PATH +GET_STAGE_LOCATION +GET_TABLE_VERSIONS +GET_TABLE_VERSION_REFS +GRANT_TO_PUBLIC_APPLICATION_ROLE +GREATEST +GREATEST_IGNORE_NULLS +GROUP +GROUPING +GROUPING_ID +H3_CELL_TO_BOUNDARY +H3_CELL_TO_CHILDREN +H3_CELL_TO_CHILDREN_STRING +H3_CELL_TO_PARENT +H3_CELL_TO_POINT +H3_COMPACT_CELLS +H3_COMPACT_CELLS_STRINGS +H3_COVERAGE +H3_COVERAGE_STRINGS +H3_GET_RESOLUTION +H3_GRID_DISK +H3_GRID_DISTANCE +H3_GRID_PATH +H3_INT_TO_STRING +H3_IS_PENTAGON +H3_IS_VALID_CELL +H3_LATLNG_TO_CELL +H3_LATLNG_TO_CELL_STRING +H3_POINT_TO_CELL +H3_POINT_TO_CELL_STRING +H3_POLYGON_TO_CELLS +H3_POLYGON_TO_CELLS_STRINGS +H3_STRING_TO_INT +H3_TRY_COVERAGE +H3_TRY_COVERAGE_STRINGS +H3_TRY_GRID_DISTANCE +H3_TRY_GRID_PATH +H3_TRY_POLYGON_TO_CELLS +H3_TRY_POLYGON_TO_CELLS_STRINGS +H3_UNCOMPACT_CELLS +H3_UNCOMPACT_CELLS_STRINGS +HARMONIC_MEAN +HASH +HASH_AGG +HAVERSINE +HEX_DECODE_BINARY +HEX_DECODE_STRING +HEX_ENCODE +HLL +HLL_ACCUMULATE +HLL_COMBINE +HLL_ESTIMATE +HLL_EXPORT +HLL_IMPORT +HOUR +ICEBERG_TABLE_FILES +ICEBERG_TABLE_FILE_REGISTRATION_HISTORY +ICEBERG_TABLE_SNAPSHOT_REFRESH_HISTORY +IDENTIFIER +IFF +IFNULL +ILIKE +IN +INFER_SCHEMA +INITCAP +INSERT +INTEGRATION +INTERPOLATE_BFILL +INTERPOLATE_FFILL +INTERPOLATE_LINEAR +INVALID_CA_POSTAL_CODE_COUNT +INVALID_CA_POSTAL_CODE_PERCENT +INVALID_CA_SIN_COUNT +INVALID_CA_SIN_PERCENT +INVALID_CH_OASI_COUNT +INVALID_CH_OASI_PERCENT +INVALID_CUSIP_COUNT +INVALID_CUSIP_PERCENT +INVALID_DE_POSTAL_CODE_COUNT +INVALID_DE_POSTAL_CODE_PERCENT +INVALID_DE_TAX_ID_COUNT +INVALID_DE_TAX_ID_PERCENT +INVALID_EMAIL_COUNT +INVALID_EMAIL_PERCENT +INVALID_ES_DNI_COUNT +INVALID_ES_DNI_PERCENT +INVALID_FIGI_COUNT +INVALID_FIGI_PERCENT +INVALID_FR_INSEE_CODE_COUNT +INVALID_FR_INSEE_CODE_PERCENT +INVALID_FR_POSTAL_CODE_COUNT +INVALID_FR_POSTAL_CODE_PERCENT +INVALID_INTEGER_COUNT +INVALID_INTEGER_PERCENT +INVALID_ISIN_COUNT +INVALID_ISIN_PERCENT +INVALID_ISO8601_FORMAT_COUNT +INVALID_ISO8601_FORMAT_PERCENT +INVALID_IT_FISCAL_CODE_COUNT +INVALID_IT_FISCAL_CODE_PERCENT +INVALID_JSON_COUNT +INVALID_JSON_PERCENT +INVALID_LATITUDE_COUNT +INVALID_LATITUDE_PERCENT +INVALID_LEI_COUNT +INVALID_LEI_PERCENT +INVALID_LONGITUDE_COUNT +INVALID_LONGITUDE_PERCENT +INVALID_NL_BSN_COUNT +INVALID_NL_BSN_PERCENT +INVALID_NUMERIC_TYPE_CAST_COUNT +INVALID_NUMERIC_TYPE_CAST_PERCENT +INVALID_PERCENTAGE_RANGE_COUNT +INVALID_PERCENTAGE_RANGE_PERCENT +INVALID_PERM_ID_COUNT +INVALID_PERM_ID_PERCENT +INVALID_PL_PESEL_COUNT +INVALID_PL_PESEL_PERCENT +INVALID_PROBABILITY_RANGE_COUNT +INVALID_PROBABILITY_RANGE_PERCENT +INVALID_SEDOL_COUNT +INVALID_SEDOL_PERCENT +INVALID_TIMESTAMP_STRING_COUNT +INVALID_TIMESTAMP_STRING_PERCENT +INVALID_TR_ID_NO_COUNT +INVALID_TR_ID_NO_PERCENT +INVALID_UK_NINO_COUNT +INVALID_UK_NINO_PERCENT +INVALID_UK_POSTAL_CODE_COUNT +INVALID_UK_POSTAL_CODE_PERCENT +INVALID_USA_PHONE_COUNT +INVALID_USA_PHONE_PERCENT +INVALID_USA_STATE_CODE_COUNT +INVALID_USA_STATE_CODE_PERCENT +INVALID_USA_ZIP_CODE_COUNT +INVALID_USA_ZIP_CODE_PERCENT +INVALID_UUID_COUNT +INVALID_UUID_PERCENT +INVOKER_ROLE +INVOKER_SHARE +IPADDRESS_TRUNC +IS_APPLICATION_ROLE_IN_SESSION +IS_ARRAY +IS_BINARY +IS_BOOLEAN +IS_CHAR +IS_DATABASE_ROLE_IN_SESSION +IS_DATE +IS_DATE_VALUE +IS_DECIMAL +IS_DOUBLE +IS_GENERATED +IS_GRANTED_TO_INVOKER_ROLE +IS_INSTANCE_ROLE_IN_SESSION +IS_INTEGER +IS_NULL_VALUE +IS_OBJECT +IS_ORGANIZATION_USER +IS_ORGANIZATION_USER_GROUP +IS_ORGANIZATION_USER_GROUP_IN_SESSION +IS_REAL +IS_ROLE_IN_SESSION +IS_TIME +IS_TIMESTAMP_LTZ +IS_TIMESTAMP_NTZ +IS_TIMESTAMP_TZ +IS_VARCHAR +JAROWINKLER_SIMILARITY +JSON_EXTRACT_PATH_TEXT +KURTOSIS +LAG +LAST_DAY +LAST_QUERY_ID +LAST_SUCCESSFUL_SCHEDULED_TIME +LAST_TRANSACTION +LAST_VALUE +LATERAL +LEAD +LEAST +LEAST_IGNORE_NULLS +LEFT +LEN +LENGTH +LIKE +LISTAGG +LISTING_REFRESH_HISTORY +LN +LOCALTIME +LOCALTIMESTAMP +LOG +LOGIN_HISTORY +LOGIN_HISTORY_BY_USER +LOWER +LPAD +LTRIM +MAE +MAP +MAPE +MAP_CAT +MAP_CONSTRUCT +MAP_CONTAINS_KEY +MAP_DELETE +MAP_ENTRIES +MAP_INSERT +MAP_KEYS +MAP_PICK +MAP_SIZE +MATCH_CONDITION +MATCH_RECOGNIZE +MATERIALIZED_VIEW_REFRESH_HISTORY +MAX +MAX_BY +MD5 +MD5_BINARY +MD5_HEX +MD5_NUMBER +MD5_NUMBER_LOWER64 +MD5_NUMBER_UPPER64 +MEAN_ERROR +MEDIAN +MIN +MINHASH +MINHASH_COMBINE +MINUTE +MIN_BY +MOD +MODE +MODEL +MODEL_MONITOR_DRIFT_METRIC +MODEL_MONITOR_PERFORMANCE_METRIC +MODEL_MONITOR_STAT_METRIC +MONTH +MONTHNAME +MONTHS_BETWEEN +MY_BUDGET_SUMMARY +NAN_COUNT +NAN_PERCENT +NEGATIVE_COUNT +NEGATIVE_PERCENT +NETWORK_RULE_REFERENCES +NEXT_DAY +NORMAL +NOTIFICATION_HISTORY +NOT_FRIDAY_COUNT +NOT_FRIDAY_PERCENT +NOT_IN_FUTURE_COUNT +NOT_IN_FUTURE_PERCENT +NOT_MONDAY_COUNT +NOT_MONDAY_PERCENT +NOT_NULL_COUNT +NOT_NULL_PERCENT +NOT_SATURDAY_COUNT +NOT_SATURDAY_PERCENT +NOT_SUNDAY_COUNT +NOT_SUNDAY_PERCENT +NOT_THURSDAY_COUNT +NOT_THURSDAY_PERCENT +NOT_TODAY_COUNT +NOT_TODAY_PERCENT +NOT_TUESDAY_COUNT +NOT_TUESDAY_PERCENT +NOT_WEDNESDAY_COUNT +NOT_WEDNESDAY_PERCENT +NOT_WEEKDAY_COUNT +NOT_WEEKDAY_PERCENT +NOT_YESTERDAY_COUNT +NOT_YESTERDAY_PERCENT +NTH_VALUE +NTILE +NULL +NULLIF +NULLIFZERO +NULL_COUNT +NULL_KEYWORD_COUNT +NULL_KEYWORD_PERCENT +NULL_PERCENT +NVL +NVL2 +OBJECT +OBJECT_AGG +OBJECT_CONSTRUCT +OBJECT_CONSTRUCT_KEEP_NULL +OBJECT_CONSTRUCT_KEEP_NULL_STRUCTURED +OBJECT_DELETE +OBJECT_INSERT +OBJECT_KEYS +OBJECT_PICK +OCTET_LENGTH +ON +ONLINE_FEATURE_TABLE_REFRESH_HISTORY +OUTLIER_COUNT +OUTLIER_IQR_COUNT +OUTLIER_IQR_PERCENT +OUTLIER_PERCENT +OUTLIER_ZSCORE_COUNT +OUTLIER_ZSCORE_PERCENT +OVER +PARSE_DOCUMENT +PARSE_IP +PARSE_JSON +PARSE_URL +PARSE_XML +PATTERN +PERCENTILE_CONT +PERCENTILE_DISC +PERCENT_RANK +PERIOD +PERIOD_BEGIN +PERIOD_CONSTRUCT +PERIOD_CONTAINS +PERIOD_END +PERIOD_EQUALS +PERIOD_IMMEDIATELY_PRECEDES +PERIOD_IMMEDIATELY_SUCCEEDS +PERIOD_INTERSECT +PERIOD_LDIFF +PERIOD_MEETS +PERIOD_OVERLAPS +PERIOD_PRECEDES +PERIOD_RDIFF +PERIOD_SUCCEEDS +PI +PIPE_USAGE_HISTORY +PIVOT +POLICY_CONTEXT +POLICY_REFERENCES +POSITION +POW +POWER +PREVIOUS_DAY +PROMPT +QUARTER +QUERY_ACCELERATION_HISTORY +QUERY_HISTORY +QUERY_HISTORY_BY_SESSION +QUERY_HISTORY_BY_USER +QUERY_HISTORY_BY_WAREHOUSE +RADIANS +RANDOM +RANDSTR +RANK +RATIO_TO_REPORT +RECOMMEND_NETWORK_POLICY +REDUCE +REGEXP_COUNT +REGEXP_INSTR +REGEXP_LIKE +REGEXP_REPLACE +REGEXP_SUBSTR +REGEXP_SUBSTR_ALL +REGISTER_EXTENSION +REGR_AVGX +REGR_AVGY +REGR_COUNT +REGR_INTERCEPT +REGR_R2 +REGR_SLOPE +REGR_SXX +REGR_SXY +REGR_SYY +REGR_VALX +REGR_VALY +REPEAT +REPLACE +REPLICATION_GROUP_DANGLING_REFERENCES +REPLICATION_GROUP_LAG_HISTORY +REPLICATION_GROUP_REFRESH_HISTORY +REPLICATION_GROUP_REFRESH_HISTORY_ALL +REPLICATION_GROUP_REFRESH_PROGRESS +REPLICATION_GROUP_REFRESH_PROGRESS_ALL +REPLICATION_GROUP_REFRESH_PROGRESS_BY_JOB +REPLICATION_GROUP_USAGE_HISTORY +REPLICATION_USAGE_HISTORY +RESAMPLE +RESET_PRIVACY_BUDGET +REST_EVENT_HISTORY +RESULT_SCAN +REVERSE +REVOKE_FROM_PUBLIC_APPLICATION_ROLE +RIGHT +RLIKE +RMSE +ROLLUP +ROUND +ROWS_IN_PAST_DAY_COUNT +ROWS_IN_PAST_DAY_PERCENT +ROWS_IN_PAST_HOUR_COUNT +ROWS_IN_PAST_HOUR_PERCENT +ROWS_IN_PAST_MONTH_COUNT +ROWS_IN_PAST_MONTH_PERCENT +ROWS_IN_PAST_WEEK_COUNT +ROWS_IN_PAST_WEEK_PERCENT +ROWS_IN_PAST_YEAR_COUNT +ROWS_IN_PAST_YEAR_PERCENT +ROW_NUMBER +RPAD +RTRIM +RTRIMMED_LENGTH +R_SQUARED +SANITIZE_WEBHOOK_CONTENT +SCHEDULED_TIME +SEARCH +SEARCH_IP +SEARCH_OPTIMIZATION_HISTORY +SEARCH_PREVIEW +SECOND +SEMANTIC_VIEW +SEMANTIC_VIEW_MATERIALIZATION_REFRESH_HISTORY +SENTIMENT +SEQ1 +SEQ2 +SEQ4 +SEQ8 +SERVERLESS_ALERT_HISTORY +SERVERLESS_TASK_HISTORY +SETS +SET_SYS_CONTEXT +SHA1 +SHA1_BINARY +SHA1_HEX +SHA2 +SHA2_BINARY +SHA2_HEX +SHOW_PYTHON_PACKAGES_DEPENDENCIES +SIGN +SIN +SINH +SKEW +SOUNDEX +SOUNDEX_P123 +SOURCE_TARGET_LAG +SPACE +SPCS_CANCEL_JOB +SPCS_GET_EVENTS +SPCS_GET_LOGS +SPCS_GET_METRICS +SPCS_WAIT_FOR +SPECIAL_CHARACTER_COUNT +SPECIAL_CHARACTER_PERCENT +SPLIT +SPLIT_PART +SPLIT_TEXT_MARKDOWN_HEADER +SPLIT_TEXT_RECURSIVE_CHARACTER +SPLIT_TO_TABLE +SQRT +SQUARE +STAGE_DIRECTORY_FILE_REGISTRATION_HISTORY +STAGE_STORAGE_USAGE_HISTORY +STARTSWITH +STDDEV +STDDEV_POP +STDDEV_SAMP +STORAGE_LIFECYCLE_POLICY_HISTORY +STRING_LENGTH_AVG +STRING_LENGTH_MAX +STRING_LENGTH_MIN +STRING_LENGTH_STDDEV +STRIP_NULL_VALUE +STRTOK +STRTOK_SPLIT_TO_TABLE +STRTOK_TO_ARRAY +ST_AREA +ST_ASBINARY +ST_ASEWKB +ST_ASEWKT +ST_ASGEOJSON +ST_ASTEXT +ST_ASWKB +ST_ASWKT +ST_AZIMUTH +ST_BUFFER +ST_CENTROID +ST_COLLECT +ST_CONTAINS +ST_COVEREDBY +ST_COVERS +ST_DIFFERENCE +ST_DIMENSION +ST_DISJOINT +ST_DISTANCE +ST_DWITHIN +ST_ENDPOINT +ST_ENVELOPE +ST_GEOGFROMEWKB +ST_GEOGFROMEWKT +ST_GEOGFROMGEOHASH +ST_GEOGFROMTEXT +ST_GEOGFROMWKB +ST_GEOGFROMWKT +ST_GEOGPOINTFROMGEOHASH +ST_GEOGRAPHYFROMEWKB +ST_GEOGRAPHYFROMEWKT +ST_GEOGRAPHYFROMTEXT +ST_GEOGRAPHYFROMWKB +ST_GEOGRAPHYFROMWKT +ST_GEOHASH +ST_GEOMETRYFROMEWKB +ST_GEOMETRYFROMEWKT +ST_GEOMETRYFROMTEXT +ST_GEOMETRYFROMWKB +ST_GEOMETRYFROMWKT +ST_GEOMFROMEWKB +ST_GEOMFROMEWKT +ST_GEOMFROMGEOHASH +ST_GEOMFROMTEXT +ST_GEOMFROMWKB +ST_GEOMFROMWKT +ST_GEOMPOINTFROMGEOHASH +ST_HAUSDORFFDISTANCE +ST_INTERPOLATE +ST_INTERSECTION +ST_INTERSECTION_AGG +ST_INTERSECTS +ST_ISVALID +ST_LENGTH +ST_MAKEGEOMPOINT +ST_MAKELINE +ST_MAKEPOINT +ST_MAKEPOLYGON +ST_MAKEPOLYGONORIENTED +ST_NPOINTS +ST_PERIMETER +ST_POINTN +ST_SETSRID +ST_SIMPLIFY +ST_SRID +ST_STARTPOINT +ST_SYMDIFFERENCE +ST_TRANSFORM +ST_UNION +ST_UNION_AGG +ST_WITHIN +ST_X +ST_XMAX +ST_XMIN +ST_Y +ST_YMAX +ST_YMIN +SUBSTR +SUBSTRING +SUM +SYSDATE +SYSTEM$ABORT_SESSION +SYSTEM$ABORT_TRANSACTION +SYSTEM$ACTIVATE_CMK_INFO +SYSTEM$ACTIVATE_CMK_INFO_POSTGRES +SYSTEM$ACTIVATE_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$ADD_EVENT +SYSTEM$ADD_PRIVATELINK_ENDPOINT_HOSTNAME +SYSTEM$ADD_REFERENCE +SYSTEM$ALLOWLIST +SYSTEM$ALLOWLIST_PRIVATELINK +SYSTEM$APPLICATION_GET_LOG_LEVEL +SYSTEM$APPLICATION_GET_METRIC_LEVEL +SYSTEM$APPLICATION_GET_TRACE_LEVEL +SYSTEM$APP_COMPATIBILITY_CHECK +SYSTEM$AUTHORIZE_PRIVATELINK +SYSTEM$AUTHORIZE_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PRIVATELINK_ACCESS +SYSTEM$AUTHORIZE_STAGE_PRIVATELINK_ACCESS +SYSTEM$AUTO_REFRESH_STATUS +SYSTEM$BEGIN_DEBUG_APPLICATION +SYSTEM$BEHAVIOR_CHANGE_BUNDLE_STATUS +SYSTEM$BLOCK_INTERNAL_STAGES_PUBLIC_ACCESS +SYSTEM$BLOCK_INTERNAL_STAGES_PUBLIC_ACCESS_WITH_EXCEPTION +SYSTEM$BLOCK_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS +SYSTEM$BLOCK_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS_WITH_EXCEPTION +SYSTEM$CANCEL_ALL_QUERIES +SYSTEM$CANCEL_CLASSIFY_SCHEMA +SYSTEM$CANCEL_QUERY +SYSTEM$CAST_MULTI_PARTY_APPROVAL_VOTE +SYSTEM$CATALOG_LINK_STATUS +SYSTEM$CKE_HASH_FUNCTION +SYSTEM$CLASSIFY +SYSTEM$CLASSIFY_SCHEMA +SYSTEM$CLEANUP_DATABASE_ROLE_GRANTS +SYSTEM$CLIENT_VERSION_INFO +SYSTEM$CLIENT_VULNERABILITY_INFO +SYSTEM$CLUSTERING_DEPTH +SYSTEM$CLUSTERING_INFORMATION +SYSTEM$CLUSTERING_RATIO +SYSTEM$COMMIT_MOVE_ORGANIZATION_ACCOUNT +SYSTEM$CONVERT_PIPES_SQS_TO_SNS +SYSTEM$CREATE_BILLING_EVENT +SYSTEM$CREATE_BILLING_EVENTS +SYSTEM$CREATE_EVALUATION_DATASET +SYSTEM$CREATE_SEMANTIC_VIEW_FROM_OSI_YAML +SYSTEM$CREATE_SEMANTIC_VIEW_FROM_OSSIE_YAML +SYSTEM$CREATE_SEMANTIC_VIEW_FROM_YAML +SYSTEM$CURRENT_USER_TASK_NAME +SYSTEM$DATABASE_REFRESH_HISTORY +SYSTEM$DATABASE_REFRESH_PROGRESS +SYSTEM$DATABASE_REFRESH_PROGRESS_BY_JOB +SYSTEM$DATA_METRIC_SCAN +SYSTEM$DBT_GET_LAST_FAILED_RUN_TARGET +SYSTEM$DBT_GET_LAST_RUN_TARGET +SYSTEM$DBT_GET_LAST_SUCCESSFUL_RUN_TARGET +SYSTEM$DEACTIVATE_CMK_INFO +SYSTEM$DEACTIVATE_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$DECODE_PAT +SYSTEM$DEMIGRATE_DBT_PROJECT +SYSTEM$DEPROVISION_PRIVATELINK_ENDPOINT +SYSTEM$DEPROVISION_PRIVATELINK_ENDPOINT_TSS +SYSTEM$DEREGISTER_CMK_INFO +SYSTEM$DEREGISTER_CMK_INFO_POSTGRES +SYSTEM$DEREGISTER_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$DESC_ICEBERG_ACCESS_IDENTITY +SYSTEM$DISABLE_BEHAVIOR_CHANGE_BUNDLE +SYSTEM$DISABLE_DATABASE_REPLICATION +SYSTEM$DISABLE_FEATURE_GROUPS +SYSTEM$DISABLE_FEATURE_GROUPS_PREVIEW +SYSTEM$DISABLE_GLOBAL_DATA_SHARING_FOR_ACCOUNT +SYSTEM$DISABLE_PREVIEW_ACCESS +SYSTEM$DISABLE_PRIVATELINK_ACCESS_ONLY +SYSTEM$ENABLE_BEHAVIOR_CHANGE_BUNDLE +SYSTEM$ENABLE_FEATURE_GROUPS +SYSTEM$ENABLE_GLOBAL_DATA_SHARING_FOR_ACCOUNT +SYSTEM$ENABLE_PREVIEW_ACCESS +SYSTEM$ENCODE_CKE_PRIMARY_KEY +SYSTEM$END_DEBUG_APPLICATION +SYSTEM$ENFORCE_PRIVATELINK_ACCESS_ONLY +SYSTEM$ESTIMATE_AUTOMATIC_CLUSTERING_COSTS +SYSTEM$ESTIMATE_QUERY_ACCELERATION +SYSTEM$ESTIMATE_SEARCH_OPTIMIZATION_COSTS +SYSTEM$EVALUATE_DATA_QUALITY_EXPECTATIONS +SYSTEM$EVALUATE_DATA_QUALITY_EXPECTATIONS_PERSIST_RESULT +SYSTEM$EXECUTE_CATALOG_OPERATION +SYSTEM$EXPLAIN_JSON_TO_TEXT +SYSTEM$EXPLAIN_PLAN_JSON +SYSTEM$EXPORT_TDS_FROM_SEMANTIC_VIEW +SYSTEM$EXTERNAL_TABLE_PIPE_STATUS +SYSTEM$FETCH_EXTERNAL_SECRET_FROM_INTEGRATION +SYSTEM$FINISH_OAUTH_FLOW +SYSTEM$GENERATE_SAML_CSR +SYSTEM$GENERATE_SCIM_ACCESS_TOKEN +SYSTEM$GET_ALERT_CONFIG +SYSTEM$GET_ALERT_TEMPLATE +SYSTEM$GET_ALL_DEFAULT_COLUMNS_OVERRIDES +SYSTEM$GET_ALL_REFERENCES +SYSTEM$GET_APPLICATION_SERVICE_LOGS +SYSTEM$GET_AWS_SNS_IAM_POLICY +SYSTEM$GET_CATALOG_LINKED_DATABASE_CONFIG +SYSTEM$GET_CLASSIFICATION_RESULT +SYSTEM$GET_CMK_AKV_CONSENT_URL +SYSTEM$GET_CMK_CONFIG +SYSTEM$GET_CMK_CONFIG_POSTGRES +SYSTEM$GET_CMK_CONFIG_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$GET_CMK_INFO +SYSTEM$GET_CMK_INFO_POSTGRES +SYSTEM$GET_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$GET_CMK_KMS_KEY_POLICY +SYSTEM$GET_COMPUTE_POOL_PENDING_MAINTENANCE +SYSTEM$GET_DBT_LOG +SYSTEM$GET_DEBUG_STATUS +SYSTEM$GET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$GET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$GET_DIRECTORY_TABLE_STATUS +SYSTEM$GET_GCP_KMS_CMK_GRANT_ACCESS_CMD +SYSTEM$GET_HASH_FOR_APPLICATION +SYSTEM$GET_ICEBERG_TABLE_INFORMATION +SYSTEM$GET_INSTANCE_FAMILY_PLACEMENT_GROUPS +SYSTEM$GET_LISTING_URL_FOR_EXTERNAL_CONSUMER +SYSTEM$GET_LOGIN_FAILURE_DETAILS +SYSTEM$GET_PREDECESSOR_RETURN_VALUE +SYSTEM$GET_PREVIEW_ACCESS_STATUS +SYSTEM$GET_PRIVATELINK +SYSTEM$GET_PRIVATELINK_AUTHORIZED_ENDPOINTS +SYSTEM$GET_PRIVATELINK_CONFIG +SYSTEM$GET_PRIVATELINK_ENDPOINTS_INFO +SYSTEM$GET_PRIVATELINK_ENDPOINT_REGISTRATIONS +SYSTEM$GET_PURCHASE_ATTRIBUTES +SYSTEM$GET_REFERENCED_OBJECT_ID_HASH +SYSTEM$GET_RESULTSET_STATUS +SYSTEM$GET_SECURITY_INTEGRATIONS_FOR_API_PROVIDER +SYSTEM$GET_SERVICE_CALLER_TOKEN_EXPIRY +SYSTEM$GET_SERVICE_DNS_DOMAIN +SYSTEM$GET_SERVICE_LOGS +SYSTEM$GET_SERVICE_STATUS +SYSTEM$GET_SNOWFLAKE_EGRESS_IP_RANGES +SYSTEM$GET_SNOWFLAKE_PLATFORM_INFO +SYSTEM$GET_STAGE_PRIVATELINK_AUTHORIZED_ENDPOINTS +SYSTEM$GET_TABLE_ARCHIVE_METADATA +SYSTEM$GET_TAG +SYSTEM$GET_TAG_ALLOWED_VALUES +SYSTEM$GET_TAG_ON_CURRENT_COLUMN +SYSTEM$GET_TAG_ON_CURRENT_TABLE +SYSTEM$GET_TASK_GRAPH_CONFIG +SYSTEM$GET_TZDB_VERSION +SYSTEM$GLOBAL_ACCOUNT_SET_PARAMETER +SYSTEM$HOLD_PRIVILEGE_ON_ACCOUNT +SYSTEM$INGEST_REBIND_PIPE +SYSTEM$INITIATE_MOVE_ORGANIZATION_ACCOUNT +SYSTEM$INTERNAL_STAGES_PUBLIC_ACCESS_STATUS +SYSTEM$ISSUE_PER_ACCOUNT_APP_SERVICE_CERTIFICATE +SYSTEM$ISSUE_PER_ACCOUNT_CERTIFICATES +SYSTEM$ISSUE_WORKLOAD_IDENTITY_FEDERATION_TOKEN +SYSTEM$IS_APPLICATION_AUTHORIZED_FOR_TELEMETRY_EVENT_SHARING +SYSTEM$IS_APPLICATION_INSTALLED_FROM_SAME_ACCOUNT +SYSTEM$IS_APPLICATION_SHARING_EVENTS_WITH_PROVIDER +SYSTEM$IS_GLOBAL_DATA_SHARING_ENABLED_FOR_ACCOUNT +SYSTEM$IS_LISTING_PURCHASED +SYSTEM$IS_LISTING_TRIAL +SYSTEM$LAST_CHANGE_COMMIT_TIME +SYSTEM$LINK_ACCOUNT_OBJECTS_BY_NAME +SYSTEM$LINK_ORGANIZATION_USER +SYSTEM$LINK_ORGANIZATION_USER_GROUP +SYSTEM$LIST_ALERT_TEMPLATES +SYSTEM$LIST_APPLICATION_RESTRICTED_FEATURES +SYSTEM$LIST_EXTERNAL_SECRETS +SYSTEM$LIST_ICEBERG_TABLES_FROM_CATALOG +SYSTEM$LIST_NAMESPACES_FROM_CATALOG +SYSTEM$LOCATE_DBT_ARCHIVE +SYSTEM$LOCATE_DBT_ARTIFACTS +SYSTEM$LOG +SYSTEM$LOG_DEBUG +SYSTEM$LOG_ERROR +SYSTEM$LOG_FATAL +SYSTEM$LOG_INFO +SYSTEM$LOG_TRACE +SYSTEM$LOG_WARN +SYSTEM$MANAGE_SEMANTIC_VIEW_MATERIALIZATIONS_FROM_YAML +SYSTEM$MIGRATE_DBT_PROJECT +SYSTEM$MIGRATE_LEGACY_EVENT_ROUTING_CONFIGURATION +SYSTEM$MIGRATE_SAML_IDP_REGISTRATION +SYSTEM$OPT_IN_INTERNAL_STAGE_NETWORK_LOGS +SYSTEM$OPT_OUT_INTERNAL_STAGE_NETWORK_LOGS +SYSTEM$OPT_OUT_MALICIOUS_IP_PROTECTION_BY_CATEGORY +SYSTEM$PARQUET_DIRECT_AUTO_REFRESH_STATUS +SYSTEM$PIPE_FORCE_RESUME +SYSTEM$PIPE_REBINDING_WITH_NOTIFICATION_CHANNEL +SYSTEM$PIPE_STATUS +SYSTEM$PREDECESSORS_STATE +SYSTEM$PROVISION_PRIVATELINK_ENDPOINT +SYSTEM$PROVISION_PRIVATELINK_ENDPOINT_TSS +SYSTEM$QUERY_REFERENCE +SYSTEM$READ_OSI_YAML_FROM_SEMANTIC_VIEW +SYSTEM$READ_OSSIE_YAML_FROM_SEMANTIC_VIEW +SYSTEM$READ_YAML_FROM_SEMANTIC_VIEW +SYSTEM$REFERENCE +SYSTEM$REGISTER_CMK_INFO +SYSTEM$REGISTER_CMK_INFO_POSTGRES +SYSTEM$REGISTER_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$REGISTER_PRIVATELINK_ENDPOINT +SYSTEM$REGISTRY_LIST_IMAGES +SYSTEM$REMOTE_APP_OPERATION +SYSTEM$REMOVE_ALL_REFERENCES +SYSTEM$REMOVE_PRIVATELINK_ENDPOINT_HOSTNAME +SYSTEM$REMOVE_REFERENCE +SYSTEM$REPORT_HEALTH_STATUS +SYSTEM$REQUEST_LISTING_AND_WAIT +SYSTEM$RESOLVE_PYTHON_PACKAGES +SYSTEM$RESTORE_PRIVATELINK_ENDPOINT +SYSTEM$RESTORE_PRIVATELINK_ENDPOINT_TSS +SYSTEM$REVOKE_PRIVATELINK +SYSTEM$REVOKE_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PRIVATELINK_ACCESS +SYSTEM$REVOKE_STAGE_PRIVATELINK_ACCESS +SYSTEM$SCHEDULE_ASYNC_REPLICATION_GROUP_REFRESH +SYSTEM$SEND_EMAIL +SYSTEM$SEND_NOTIFICATIONS_TO_CATALOG +SYSTEM$SEND_SNOWFLAKE_NOTIFICATION +SYSTEM$SET_APPLICATION_RESTRICTED_FEATURE_ACCESS +SYSTEM$SET_CATALOG_INTEGRATION +SYSTEM$SET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$SET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$SET_EVENT_SHARING_ACCOUNT_FOR_REGION +SYSTEM$SET_PRIVATELINK_ENDPOINT_HOSTNAME +SYSTEM$SET_REFERENCE +SYSTEM$SET_RETURN_VALUE +SYSTEM$SET_ROW_TIMESTAMP_ON_ALL_SUPPORTED_TABLES +SYSTEM$SET_SPAN_ATTRIBUTES +SYSTEM$SHOW_ACTIVE_BEHAVIOR_CHANGE_BUNDLES +SYSTEM$SHOW_BUDGETS_FOR_RESOURCE +SYSTEM$SHOW_BUDGETS_IN_ACCOUNT +SYSTEM$SHOW_BUDGET_SHARED_RESOURCE_CANDIDATES +SYSTEM$SHOW_DYNAMIC_TABLES_CREATED_FOR_RESHARING +SYSTEM$SHOW_EVENT_SHARING_ACCOUNTS +SYSTEM$SHOW_MOVE_ORGANIZATION_ACCOUNT_STATUS +SYSTEM$SHOW_OAUTH_CLIENT_SECRETS +SYSTEM$SHOW_SENSITIVE_DATA_MONITORED_ENTITIES +SYSTEM$SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS_STATUS +SYSTEM$SNOWPIPE_STREAMING_UPDATE_CHANNEL_OFFSET_TOKEN +SYSTEM$START_OAUTH_FLOW +SYSTEM$START_USER_EMAIL_VERIFICATION +SYSTEM$STREAM_BACKLOG +SYSTEM$STREAM_GET_TABLE_TIMESTAMP +SYSTEM$STREAM_HAS_DATA +SYSTEM$SUPPORTED_DBT_VERSIONS +SYSTEM$TAG_VALUE_CONTAINS +SYSTEM$TASK_COMPLETION_STATE +SYSTEM$TASK_DEPENDENTS_ENABLE +SYSTEM$TASK_RUNTIME_INFO +SYSTEM$TRIGGER_LISTING_REFRESH +SYSTEM$TYPEOF +SYSTEM$UNBLOCK_INTERNAL_STAGES_PUBLIC_ACCESS +SYSTEM$UNBLOCK_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS +SYSTEM$UNLINK_ORGANIZATION_USER +SYSTEM$UNLINK_ORGANIZATION_USER_GROUP +SYSTEM$UNMANAGE_ENTITY_FROM_DCM_PROJECT +SYSTEM$UNMANAGE_FROM_DCM_PROJECT +SYSTEM$UNREGISTER_PRIVATELINK_ENDPOINT +SYSTEM$UNSET_DEFAULT_COLUMNS_OVERRIDE_FOR_SHOW_COMMAND +SYSTEM$UNSET_DEFAULT_COLUMNS_OVERRIDE_FOR_SYSTEM_OBJECT +SYSTEM$UNSET_EVENT_SHARING_ACCOUNT_FOR_REGION +SYSTEM$UNVERIFY_DNS_DOMAIN +SYSTEM$UPDATE_MULTI_PARTY_APPROVAL_JUSTIFICATION +SYSTEM$USER_TASK_CANCEL_ONGOING_EXECUTIONS +SYSTEM$VALIDATE_STORAGE_INTEGRATION +SYSTEM$VERIFY_CATALOG_INTEGRATION +SYSTEM$VERIFY_CMK_INFO +SYSTEM$VERIFY_CMK_INFO_POSTGRES +SYSTEM$VERIFY_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME +SYSTEM$VERIFY_DNS_DOMAIN +SYSTEM$VERIFY_EXTERNAL_OAUTH_TOKEN +SYSTEM$VERIFY_EXTERNAL_SECRET_INTEGRATION +SYSTEM$VERIFY_EXTERNAL_VOLUME +SYSTEM$WAIT +SYSTEM$WAIT_FOR_OPENFLOW_CONNECTOR_STATUS +SYSTEM$WAIT_FOR_OPENFLOW_DEPLOYMENT_STATUS +SYSTEM$WAIT_FOR_OPENFLOW_RUNTIME_STATUS +SYSTEM$WAIT_FOR_SERVICES +SYSTEM$WAIT_FOR_STABLE_OPENFLOW_CONNECTORS +SYSTEM$WAIT_FOR_STABLE_OPENFLOW_DEPLOYMENTS +SYSTEM$WAIT_FOR_STABLE_OPENFLOW_RUNTIMES +SYSTEM$WHITELIST +SYSTEM$WHITELIST_PRIVATELINK +SYSTIMESTAMP +SYS_CONTEXT +TABLE +TAG_REFERENCES +TAG_REFERENCES_ALL_COLUMNS +TAG_REFERENCES_WITH_LINEAGE +TAN +TANH +TASK_DEPENDENTS +TASK_GRAPH_CHANGES +TASK_HISTORY +TEXT_HTML +TEXT_PLAIN +THREAD_MESSAGES +TIME +TIMEADD +TIMEDIFF +TIMESTAMPADD +TIMESTAMPDIFF +TIMESTAMP_FROM_PARTS +TIMESTAMP_LTZ_FROM_PARTS +TIMESTAMP_NTZ_FROM_PARTS +TIMESTAMP_TZ_FROM_PARTS +TIME_FROM_PARTS +TIME_SLICE +TM9 +TO_ARRAY +TO_BINARY +TO_BOOLEAN +TO_CHAR +TO_DATE +TO_DECFLOAT +TO_DECIMAL +TO_DOUBLE +TO_FILE +TO_GEOGRAPHY +TO_GEOMETRY +TO_IPADDRESS +TO_JSON +TO_NUMBER +TO_NUMERIC +TO_OBJECT +TO_QUERY +TO_TIME +TO_TIMESTAMP +TO_TIMESTAMP_LTZ +TO_TIMESTAMP_NTZ +TO_TIMESTAMP_TZ +TO_UUID +TO_VARCHAR +TO_VARIANT +TO_XML +TRANSFORM +TRANSLATE +TRIM +TRUE_COUNT +TRUE_PERCENT +TRUNC +TRUNCATE +TRY_BASE64_DECODE_BINARY +TRY_BASE64_DECODE_STRING +TRY_CAST +TRY_COMPLETE +TRY_DECRYPT +TRY_DECRYPT_RAW +TRY_HEX_DECODE_BINARY +TRY_HEX_DECODE_STRING +TRY_PARSE_JSON +TRY_TO_BINARY +TRY_TO_BOOLEAN +TRY_TO_DATE +TRY_TO_DECFLOAT +TRY_TO_DECIMAL +TRY_TO_DOUBLE +TRY_TO_FILE +TRY_TO_GEOGRAPHY +TRY_TO_GEOMETRY +TRY_TO_IPADDRESS +TRY_TO_NUMBER +TRY_TO_NUMERIC +TRY_TO_TIME +TRY_TO_TIMESTAMP +TRY_TO_TIMESTAMP_LTZ +TRY_TO_TIMESTAMP_NTZ +TRY_TO_TIMESTAMP_TZ +TRY_TO_UUID +TYPEOF +UNICODE +UNIFORM +UNIQUE_COUNT +UNIQUE_PERCENT +UNIX_ZERO_COUNT +UNIX_ZERO_PERCENT +UNTRIMMED_STRING_COUNT +UNTRIMMED_STRING_PERCENT +UPDATE_AUTO_ENABLEMENT_STATUS +UPPER +USING +UUID_STRING +VALIDATE +VALIDATE_PIPE_LOAD +VALUES +VARIANCE +VARIANCE_POP +VAR_POP +VAR_SAMP +VECTOR +VECTOR_AVG +VECTOR_COSINE_SIMILARITY +VECTOR_INNER_PRODUCT +VECTOR_L1_DISTANCE +VECTOR_L2_DISTANCE +VECTOR_MAX +VECTOR_MIN +VECTOR_NORMALIZE +VECTOR_SUM +VECTOR_TRUNCATE +WAREHOUSE_LOAD_HISTORY +WAREHOUSE_METERING_HISTORY +WEEK +WEEKISO +WEEKOFYEAR +WIDTH_BUCKET +XMLGET +YEAR +YEAROFWEEK +YEAROFWEEKISO +ZEROIFNULL +ZERO_COUNT +ZERO_PERCENT +ZIPF diff --git a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs index 748091fa6..1d6004cc2 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/helpers.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/helpers.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; +use std::path::Path; use crate::lineage::models::NullabilityRule; use crate::lineage::models::{Nullability, SchemaColumn, SourceColumn}; @@ -357,3 +358,111 @@ const CANONICAL_DUCKDB_NAMES: [(&[&str], &str); 19] = [ (&["TIMETZ", "TIME WITH TIME ZONE"], "TIMETZ"), (&["BLOB", "BYTEA", "VARBINARY"], "BLOB"), ]; + +/// Every upper-case name the syntax catalogue's function, expression and query grammars call. +pub(crate) fn catalogue_function_names() -> BTreeSet { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../coverage/syntax/catalogue"); + std::fs::read_dir(root) + .expect("the catalogue reads") + .map(|entry| entry.expect("a catalogue file").path()) + .map(|path| std::fs::read_to_string(path).expect("a catalogue file reads")) + .map(|text| toml::from_str::(&text).expect("a catalogue file parses")) + .flat_map(|catalogue| file_function_names(&catalogue)) + .collect() +} + +/// The names one catalogue file's call grammars and function definitions name. +fn file_function_names(catalogue: &toml::Table) -> Vec { + let entries = |key: &str| -> Vec { + catalogue + .get(key) + .and_then(toml::Value::as_array) + .cloned() + .unwrap_or_default() + }; + let text = |entry: &toml::Value, key: &str| -> String { + entry + .get(key) + .and_then(toml::Value::as_str) + .unwrap_or_default() + .to_owned() + }; + let called: Vec = entries("form") + .iter() + .filter(|form| CALLING_FAMILIES.contains(&text(form, "family").as_str())) + .flat_map(|form| { + form.get("grammar") + .and_then(toml::Value::as_array) + .cloned() + .unwrap_or_default() + }) + .flat_map(|grammar| called_names(grammar.as_str().unwrap_or_default())) + .collect(); + let alternatives: Vec = entries("definition") + .iter() + .filter(|definition| text(definition, "source").starts_with(FUNCTION_PAGE_PREFIX)) + .flat_map(|definition| definition_names(&text(definition, "grammar"))) + .collect(); + called.into_iter().chain(alternatives).collect() +} + +/// The names a `name ::= A | B` definition offers as alternatives. +fn definition_names(grammar: &str) -> Vec { + grammar + .split_once(DEFINES) + .map(|(_, alternatives)| alternatives) + .unwrap_or_default() + .split(ALTERNATIVE) + .map(str::trim) + .filter(|alternative| upper_name(alternative)) + .map(str::to_owned) + .collect() +} + +/// The upper-case names a grammar line calls: each directly followed by `(`. +fn called_names(grammar: &str) -> Vec { + grammar + .match_indices(CALL_OPEN) + .filter_map(|(at, _)| called_before(&grammar[..at])) + .collect() +} + +/// The upper-case name that ends `before`, ignoring trailing spaces, when there is one. +fn called_before(before: &str) -> Option { + let before: &str = before.trim_end(); + let start: usize = before + .char_indices() + .rev() + .find(|(_, character)| !name_character(*character)) + .map_or(0, |(at, character)| at + character.len_utf8()); + let name: &str = &before[start..]; + upper_name(name).then(|| name.to_owned()) +} + +/// Whether `character` can be part of a Snowflake name as the catalogue writes it. +fn name_character(character: char) -> bool { + character.is_ascii_alphanumeric() || NAME_SYMBOLS.contains(&character) +} + +/// Whether `name` is an upper-case identifier, as the catalogue writes Snowflake's names. +fn upper_name(name: &str) -> bool { + name.starts_with(|character: char| character.is_ascii_uppercase()) + && name.chars().all(|character| { + character.is_ascii_uppercase() + || character.is_ascii_digit() + || NAME_SYMBOLS.contains(&character) + }) +} + +/// The catalogue families whose grammars name functions. +const CALLING_FAMILIES: [&str; 3] = ["functions", "expressions", "query"]; +/// How the catalogue's source ids for function pages start. +const FUNCTION_PAGE_PREFIX: &str = "doc.functions."; +/// What separates a definition's name from its alternatives. +const DEFINES: &str = "::="; +/// What separates a definition's alternatives. +const ALTERNATIVE: char = '|'; +/// What follows a called name. +const CALL_OPEN: char = '('; +/// The symbols a Snowflake name may hold besides letters and digits. +const NAME_SYMBOLS: [char; 2] = ['_', '$']; diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_statement_findings.rs b/crates/sqlquack-semantics/src/lineage/tests/test_statement_findings.rs index df4c3c577..3ed13ea56 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_statement_findings.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_statement_findings.rs @@ -2,11 +2,19 @@ use sqlquack_syntax::{CallMode, Dialect, ParseOptions, parse_statements_with}; use crate::dialects::constants::{DUCKDB, SNOWFLAKE}; use crate::lineage::_helpers::findings::statement::statement_findings; -use crate::lineage::models::LineageContext; +use crate::lineage::constants::SNOWFLAKE_FUNCTION_NAMES; +use crate::lineage::models::{LineageContext, SchemaCatalog}; use crate::lineage::tests::helpers::{ - duckdb_shop_schema, found_findings, owned_findings, shop_schema, + catalogue_function_names, duckdb_shop_schema, found_findings, owned_findings, shop_schema, }; -use crate::lineage::tests::test_types::QueryFindingsTestCase; +use crate::lineage::tests::test_types::{ + ClosedSchemaTestCase, FunctionListTestCase, QueryFindingsTestCase, +}; +use crate::types::models::{ + ColumnSpec, FunctionLanguage, FunctionReturns, FunctionSpec, FunctionTraits, Schema, +}; +use sqlquack_snowflake::typing::models::SnowflakeType; +use std::collections::BTreeSet; #[test] fn given_a_query_when_resolving_its_names_then_unknown_ambiguous_and_unqualifiable_ones_are_found() @@ -380,3 +388,101 @@ fn given_a_duckdb_lateral_star_over_an_earlier_item_when_resolving_names_then_it ); } } + +#[test] +fn given_a_closed_schema_when_checking_names_then_undeclared_relations_and_functions_are_found() { + let mut schema = Schema::default(); + schema.relations.insert( + "ORDERS".to_owned(), + vec![ + ColumnSpec::new( + "ORDER_ID", + SnowflakeType::Number { + precision: 38, + scale: 0, + }, + ), + ColumnSpec::new("NOTE", SnowflakeType::Varchar), + ], + ); + schema.functions.insert( + "IS_GIFT".to_owned(), + FunctionSpec { + language: FunctionLanguage::Python, + arguments: vec![ColumnSpec::new("NOTE", SnowflakeType::Varchar)], + returns: FunctionReturns::Scalar(SnowflakeType::Boolean), + traits: FunctionTraits::default(), + }, + ); + let catalog = SchemaCatalog::new(&schema); + let test_cases = [ + ClosedSchemaTestCase { + description: "a relation the schema does not declare", + sql: "SELECT order_id FROM orderz", + expected_codes: &["E200"], + }, + ClosedSchemaTestCase { + description: "a CTE is no stored relation", + sql: "WITH recent AS (SELECT order_id FROM orders) SELECT order_id FROM recent", + expected_codes: &[], + }, + ClosedSchemaTestCase { + description: "a recursive CTE reads itself", + sql: "WITH RECURSIVE n AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM n WHERE i < 3) SELECT i FROM n", + expected_codes: &[], + }, + ClosedSchemaTestCase { + description: "a function neither Snowflake nor the schema declares", + sql: "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders", + expected_codes: &["E202"], + }, + ClosedSchemaTestCase { + description: "a function the schema declares, called in lower case", + sql: "SELECT is_gift(note) AS g FROM orders", + expected_codes: &[], + }, + ClosedSchemaTestCase { + description: "a Snowflake function named only in a documented alternative", + sql: "SELECT TO_TIMESTAMP_NTZ(note) AS t FROM orders", + expected_codes: &[], + }, + ]; + for test_case in test_cases { + let codes: Vec<&str> = + found_findings(test_case.sql, (Dialect::Snowflake, &SNOWFLAKE, &catalog)) + .into_iter() + .map(|(code, _, _)| code) + .collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + } +} + +#[test] +fn given_the_syntax_catalogue_when_listing_snowflake_functions_then_the_list_matches_it() { + let test_cases = [FunctionListTestCase { + description: "snowflake_functions.txt lists exactly the names the catalogue's grammars call", + expected_differences: (Vec::new(), Vec::new()), + }]; + let catalogued: BTreeSet = catalogue_function_names(); + let listed: BTreeSet = SNOWFLAKE_FUNCTION_NAMES + .lines() + .map(str::to_owned) + .collect(); + for test_case in test_cases { + assert_eq!( + ( + catalogued + .difference(&listed) + .cloned() + .collect::>(), + listed + .difference(&catalogued) + .cloned() + .collect::>() + ), + test_case.expected_differences, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs index 85a068f27..a60a1f923 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs @@ -149,3 +149,14 @@ pub(crate) struct QueryFindingsTestCase { pub(crate) sql: &'static str, pub(crate) expected_findings: &'static [(&'static str, &'static str, usize)], } + +pub(crate) struct ClosedSchemaTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_codes: &'static [&'static str], +} + +pub(crate) struct FunctionListTestCase { + pub(crate) description: &'static str, + pub(crate) expected_differences: (Vec, Vec), +} diff --git a/crates/sqlquack-semantics/src/lineage/types.rs b/crates/sqlquack-semantics/src/lineage/types.rs index 0b5b80712..58ed453f7 100644 --- a/crates/sqlquack-semantics/src/lineage/types.rs +++ b/crates/sqlquack-semantics/src/lineage/types.rs @@ -40,6 +40,11 @@ pub trait LineageSchema: std::fmt::Debug + Sync { None } + /// Whether the schema declares every relation and function the SQL may name, so an unknown one is wrong. + fn closed(&self) -> bool { + false + } + /// Whether the project declares the type `name`, which a cast may name. fn known_type(&self, name: &str) -> bool { let _ = name; diff --git a/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs b/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs index 55bfc3698..5daee7f76 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/types/data_types.rs @@ -1011,12 +1011,17 @@ impl Parser { if let Some(error) = self.recursion.error() { return error; } - let span = self - .tokens - .get(self.current) - .or_else(|| self.tokens.last()) - .map(|token| token.span) - .unwrap_or_default(); + let span = match self.tokens.get(self.current) { + Some(token) => token.span, + None => self + .tokens + .last() + .map(|token| Span { + start: token.span.end, + ..token.span + }) + .unwrap_or_default(), + }; Error::parse(message, span) } diff --git a/crates/sqlquack/src/analysis/constants.rs b/crates/sqlquack/src/analysis/constants.rs index de8af4eab..585735e1a 100644 --- a/crates/sqlquack/src/analysis/constants.rs +++ b/crates/sqlquack/src/analysis/constants.rs @@ -3,6 +3,11 @@ /// The key recorded for a refusal that names no construct. pub const STATEMENT_REFUSAL: &str = "statement"; +/// The name check's code for a relation the schema does not declare. +pub const UNKNOWN_TABLE_FINDING: &str = "E200"; +/// The name check's code for a call to a function neither Snowflake nor the schema declares. +pub const UNKNOWN_FUNCTION_FINDING: &str = "E202"; + /// The column a scalar SQL function body is converted as. pub const SCALAR_RESULT_COLUMN: &str = "RESULT"; diff --git a/crates/sqlquack/src/analysis/main/check_source.rs b/crates/sqlquack/src/analysis/main/check_source.rs index 2006cf6cc..23b29c55e 100644 --- a/crates/sqlquack/src/analysis/main/check_source.rs +++ b/crates/sqlquack/src/analysis/main/check_source.rs @@ -1,24 +1,78 @@ -//! `check_source`: what is wrong with some Snowflake SQL before any schema or conversion. +//! `check_source`: what is wrong with some Snowflake SQL before any conversion. +use sqlquack_core::classification::models::{Construct, ConstructFamily}; +use sqlquack_core::diagnostics::constants::UNKNOWN_FUNCTION; use sqlquack_core::diagnostics::main::diagnose::diagnose; use sqlquack_core::diagnostics::models::Diagnostic; use sqlquack_core::parsing::main::parse_source::parse_source; use sqlquack_core::parsing::models::{SourceDialect, SourceLines}; use sqlquack_core::refusals::main::refuse_unparsed::refuse_unparsed; use sqlquack_core::refusals::main::snowflake_rejections::snowflake_rejections; +use sqlquack_core::refusals::models::{Refusal, RefusalCategory}; +use sqlquack_semantics::dialects::constants::SNOWFLAKE; +use sqlquack_semantics::lineage::models::{LineageContext, NameFinding, SchemaCatalog}; +use sqlquack_semantics::query_name_findings; +use sqlquack_semantics::types::models::Schema; +use sqlquack_syntax::{Dialect, ParseOptions}; -/// The parse failure, or every construct Snowflake would reject; empty when the SQL is sound. -pub fn check_source(sql: &str) -> Result, String> { +use crate::analysis::constants::{UNKNOWN_FUNCTION_FINDING, UNKNOWN_TABLE_FINDING}; + +/// The parse failure, what Snowflake would reject and, with `schema`, the names it lacks. +pub fn check_source(sql: &str, schema: Option<&Schema>) -> Result, String> { let lines = SourceLines::new(sql); let refusals = match parse_source(sql, SourceDialect::Snowflake) { Ok(statements) if statements.is_empty() => { return Err("the input holds no SQL statement".to_owned()); } - Ok(statements) => statements - .iter() - .flat_map(|statement| snowflake_rejections(statement, (sql, &lines))) - .collect(), + Ok(statements) => { + let mut refusals: Vec = statements + .iter() + .flat_map(|statement| snowflake_rejections(statement, (sql, &lines))) + .collect(); + if let (Some(schema), 1) = (schema, statements.len()) { + refusals.extend(undeclared_names(sql, schema, &lines)); + } + refusals + } Err(failure) => vec![refuse_unparsed(&failure, &lines)], }; Ok(diagnose(&refusals, sql)) } + +/// The relations and functions the one statement in `sql` reads that `schema` does not declare. +fn undeclared_names(sql: &str, schema: &Schema, lines: &SourceLines) -> Vec { + let catalog = SchemaCatalog::new(schema); + let context = LineageContext { + profile: &SNOWFLAKE, + schema: &catalog, + sql: Some(sql), + }; + query_name_findings(sql, Dialect::Snowflake, &ParseOptions::default(), context) + .unwrap_or_default() + .into_iter() + .filter_map(|finding| undeclared(finding, lines)) + .collect() +} + +/// The refusal for an unknown relation or function finding; other findings are not checked here. +fn undeclared(finding: NameFinding, lines: &SourceLines) -> Option { + let location = finding + .span + .map(|span| lines.location(span.start, span.end)); + let written = finding.written; + match finding.code { + UNKNOWN_TABLE_FINDING => Some(Refusal { + construct: None, + location, + category: RefusalCategory::UnresolvedName, + reason: format!("relation `{written}` is not in the schema"), + }), + UNKNOWN_FUNCTION_FINDING => Some(Refusal { + construct: Some(Construct::new(ConstructFamily::Syntax, UNKNOWN_FUNCTION)), + location, + category: RefusalCategory::SnowflakeRejects, + reason: format!("no Snowflake or schema function is named `{written}`"), + }), + _ => None, + } +} diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index 4959566e3..4c906b469 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -32,6 +32,7 @@ use crate::analysis::models::PipelineConverter; use crate::analysis::tests::test_types::{ AnalysisSummary, CollectedSummary, DivergenceModeOutcome, EverydayCorpus, EverydayQuery, }; +use crate::cli::_helpers::schema::load_schema; use crate::cli::main::convert_report::convert_report; use crate::cli::models::{ConvertArguments, OutputArguments, OutputFormat, PolicyArguments}; use sqlquack_rules::session::main::confirm_utc_session::confirm_utc_session; @@ -928,7 +929,11 @@ fn everyday_arguments(root: &std::path::Path, builtin_policy: bool) -> ConvertAr /// What SQLQuack does with `sql`: its first check diagnostic, first refusal, or `translated`. fn everyday_outcome(sql: &str, arguments: &ConvertArguments) -> String { - let diagnostics = check_source(sql).expect("the query holds a statement"); + let schema = arguments + .schema + .as_deref() + .map(|path| load_schema(path).expect("the corpus schema loads")); + let diagnostics = check_source(sql, schema.as_ref()).expect("the query holds a statement"); diagnostics.first().map_or_else( || { let report = convert_report(arguments, sql).expect("the query converts or refuses"); diff --git a/crates/sqlquack/src/analysis/tests/test_locations.rs b/crates/sqlquack/src/analysis/tests/test_locations.rs index b76037a05..8dae61b1f 100644 --- a/crates/sqlquack/src/analysis/tests/test_locations.rs +++ b/crates/sqlquack/src/analysis/tests/test_locations.rs @@ -89,11 +89,11 @@ fn given_refused_sql_when_analyzing_then_each_refusal_is_placed_at_its_construct expected_places: vec![(RefusalCategory::NotExpressible, "NOSUCHFN(amount)", [2, 3])], }, LocationTestCase { - description: "SQL that does not parse is placed at the failure", + description: "SQL that does not parse at end of input is placed at its end", sql: "SELECT amount\nFROM orders WHERE (", declared: None, budget: None, - expected_places: vec![(RefusalCategory::ParseFidelity, "(", [2, 19])], + expected_places: vec![(RefusalCategory::ParseFidelity, "", [2, 20])], }, LocationTestCase { description: "a join no rule converts is placed at the join", diff --git a/crates/sqlquack/src/cli/_helpers/convert_command.rs b/crates/sqlquack/src/cli/_helpers/convert_command.rs index b747fc587..057bac83b 100644 --- a/crates/sqlquack/src/cli/_helpers/convert_command.rs +++ b/crates/sqlquack/src/cli/_helpers/convert_command.rs @@ -28,6 +28,7 @@ use crate::analysis::main::refused_construct::refused_construct; use crate::analysis::main::with_overrides::with_overrides; use crate::analysis::models::Analysis; use crate::cli::_helpers::evidence::caller_choices; +use crate::cli::_helpers::schema::load_schema; use crate::cli::models::{ CheckArguments, CheckReport, ConvertArguments, ConvertRefusal, ConvertReport, ExplainArguments, OverrideListing, @@ -178,14 +179,22 @@ pub(crate) fn check_command( ) -> ExitCode { let _ = writeln!(stderr, "sqlquack check: checking…"); let path = arguments.path.as_deref(); - let (sql, diagnostics) = - match read_sql(path).and_then(|sql| check_source(&sql).map(|found| (sql, found))) { - Ok(checked) => checked, - Err(error) => { - let _ = writeln!(stderr, "sqlquack check: failed: {error}"); - return ExitCode::from(2); - } - }; + let schema = match arguments.schema.as_deref().map(load_schema).transpose() { + Ok(schema) => schema, + Err(error) => { + let _ = writeln!(stderr, "sqlquack check: failed: {error}"); + return ExitCode::from(2); + } + }; + let (sql, diagnostics) = match read_sql(path) + .and_then(|sql| check_source(&sql, schema.as_ref()).map(|found| (sql, found))) + { + Ok(checked) => checked, + Err(error) => { + let _ = writeln!(stderr, "sqlquack check: failed: {error}"); + return ExitCode::from(2); + } + }; let report = CheckReport { ok: diagnostics.is_empty(), diagnostics, diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs index 4435b8f8d..f0accbca9 100644 --- a/crates/sqlquack/src/cli/models.rs +++ b/crates/sqlquack/src/cli/models.rs @@ -81,6 +81,9 @@ pub struct CompareArguments { pub struct CheckArguments { /// The SQL file; standard input when absent or `-`. pub path: Option, + /// A schema file; with one, relations and functions it does not declare are reported. + #[arg(long)] + pub schema: Option, /// How to print the report on standard output. #[command(flatten)] pub output: OutputArguments, diff --git a/crates/sqlquack/tests/cli/helpers.rs b/crates/sqlquack/tests/cli/helpers.rs index 1408963ba..71c51136c 100644 --- a/crates/sqlquack/tests/cli/helpers.rs +++ b/crates/sqlquack/tests/cli/helpers.rs @@ -29,6 +29,12 @@ fn copy_files(from: &Path, to: &Path) { } } +/// The everyday corpus's schema file. +pub(crate) const EVERYDAY_SCHEMA: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../corpora/everyday/schema.json" +); + /// Runs a command in a copy of the repository's policy and cases, with no recorded results. pub(crate) fn run_on_repository_cases(arguments: &[&str], mention: &str) -> CommandOutcome { let directory = tempfile::tempdir().expect("temporary directory"); diff --git a/crates/sqlquack/tests/cli/test_check.rs b/crates/sqlquack/tests/cli/test_check.rs index 1b43e592a..9cc427be7 100644 --- a/crates/sqlquack/tests/cli/test_check.rs +++ b/crates/sqlquack/tests/cli/test_check.rs @@ -1,4 +1,4 @@ -use crate::helpers::run_with_input; +use crate::helpers::{EVERYDAY_SCHEMA, run_with_input}; use crate::test_types::CheckCommandTestCase; #[test] @@ -76,6 +76,38 @@ fn given_snowflake_sql_when_checked_then_problems_are_coded_and_the_exit_status_ expected_stdout_prefix: "", expected_stderr_mention: "Expected a statement, not an expression", }, + CheckCommandTestCase { + description: "with a schema, a relation it does not declare is SQ0004 at the name", + arguments: &["check", "--schema", EVERYDAY_SCHEMA], + sql: "SELECT order_id FROM orderz", + expected_exit_code: Some(1), + expected_stdout_prefix: "", + expected_stderr_mention: " | ^^^^^^ relation `orderz` is not in the schema", + }, + CheckCommandTestCase { + description: "without a schema, relations are not checked", + arguments: &["check"], + sql: "SELECT order_id FROM orderz", + expected_exit_code: Some(0), + expected_stdout_prefix: "", + expected_stderr_mention: "sqlquack check: done, no problems", + }, + CheckCommandTestCase { + description: "with a schema, a function neither Snowflake nor the schema declares is SQ0205", + arguments: &["check", "--schema", EVERYDAY_SCHEMA], + sql: "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders", + expected_exit_code: Some(1), + expected_stdout_prefix: "", + expected_stderr_mention: "error[SQ0205]: Snowflake has no function of this name", + }, + CheckCommandTestCase { + description: "a recursive CTE reading itself is no unknown relation", + arguments: &["check", "--schema", EVERYDAY_SCHEMA], + sql: "WITH RECURSIVE n AS (SELECT 1 AS i UNION ALL SELECT i + 1 FROM n WHERE i < 10) SELECT i FROM n", + expected_exit_code: Some(0), + expected_stdout_prefix: "", + expected_stderr_mention: "sqlquack check: done, no problems", + }, CheckCommandTestCase { description: "input holding no statement cannot be checked and exits 2", arguments: &["check", "-"], diff --git a/crates/sqlquack/tests/cli/test_convert.rs b/crates/sqlquack/tests/cli/test_convert.rs index da3d1cd29..f3e00713c 100644 --- a/crates/sqlquack/tests/cli/test_convert.rs +++ b/crates/sqlquack/tests/cli/test_convert.rs @@ -1,7 +1,8 @@ use crate::helpers::{ - convert_outcome, convert_streams, converted_column_names, converted_first_value, - converted_run_error, converted_value_over, override_failure, override_outcome, - policy_free_exit_code, python_outcome, python_report, python_rows, run_convert, run_with_input, + EVERYDAY_SCHEMA, convert_outcome, convert_streams, converted_column_names, + converted_first_value, converted_run_error, converted_value_over, override_failure, + override_outcome, policy_free_exit_code, python_outcome, python_report, python_rows, + run_convert, run_with_input, }; use crate::test_types::{ ColumnNameTestCase, ConvertDiagnosticTestCase, ConvertOutcome, ConvertTestCase, @@ -1273,13 +1274,25 @@ fn given_a_union_of_a_float_variant_and_a_json_decimal_when_running_then_it_rais fn given_refused_sql_when_converting_then_text_and_json_carry_the_same_coded_diagnostic() { let test_cases = [ ConvertDiagnosticTestCase { - description: "a parse failure is rendered at the parser's position with its code", + description: "a parse failure is rendered at the unclosed parenthesis with its code", sql: "SELECT a,\n FROM orders WHERE (b = 1\n", flags: &[], - expected_text: "error[SQ0001]: the SQL does not parse\n --> :2:26\n |\n2 | FROM \ - orders WHERE (b = 1\n | ^ Expected RParen, got end of input \ - ('') near [b = 1]\nsqlquack convert: refused, 1 refusal under the built-in default\n", - expected_json: ("SQ0001", "parse_fidelity", 2, 26), + expected_text: "error[SQ0001]: the SQL does not parse\n --> :2:21\n |\n2 | FROM \ + orders WHERE (b = 1\n | ^ this `(` is never closed\n |\n = help: \ + add the missing `)` where the bracketed part ends\nsqlquack convert: refused, 1 refusal \ + under the built-in default\n", + expected_json: ("SQ0001", "parse_fidelity", 2, 21), + }, + ConvertDiagnosticTestCase { + description: "a double-quoted name used as a value suggests a string literal", + sql: "SELECT order_id FROM orders WHERE status = \"shipped\"", + flags: &["--schema", EVERYDAY_SCHEMA], + expected_text: "error[SQ0004]: a name does not resolve\n --> :1:44\n |\n1 | SELECT \ + order_id FROM orders WHERE status = \"shipped\"\n |\ + \x20 ^^^^^^^^^ column `shipped` does not \ + resolve\n |\n = help: use single quotes for a string literal: 'shipped'\n\ + sqlquack convert: refused, 1 refusal under the built-in default\n", + expected_json: ("SQ0004", "unresolved_name", 1, 44), }, ConvertDiagnosticTestCase { description: "an unknown relation is underlined with the fix as help", diff --git a/website/public/playground/playground.js b/website/public/playground/playground.js index ff1099098..ad5b7d61a 100644 --- a/website/public/playground/playground.js +++ b/website/public/playground/playground.js @@ -129,7 +129,7 @@ async function run() { const checks = report.checks; add(checks.diagnostics, 'error'); const findings = (checks.findings ?? []).map((finding) => { - const text = rendered({ code: finding.code, message: finding.message, span: finding.span }, chars).replace('class="bad"', 'class="warn"'); + const text = rendered({ code: finding.code, message: finding.message, span: finding.span, help: finding.help }, chars).replace('class="bad"', 'class="warn"'); spans.push({ kind: 'warn', span: finding.span, text }); return terminal(text); }); diff --git a/website/src/content/docs/docs/cli/check.mdx b/website/src/content/docs/docs/cli/check.mdx index a627b6a31..403dda1b9 100644 --- a/website/src/content/docs/docs/cli/check.mdx +++ b/website/src/content/docs/docs/cli/check.mdx @@ -5,8 +5,9 @@ sidebar: label: "check" --- -`sqlquack check` parses Snowflake SQL and reports what Snowflake would reject. It needs no schema -and no policy, and it converts nothing. +`sqlquack check` parses Snowflake SQL and reports what Snowflake would reject. It needs no policy +and converts nothing. A schema is optional: with one, `check` also reports relations and functions +the schema does not declare. ## Usage @@ -35,17 +36,22 @@ sqlquack check: found 1 problem ## What it reports -- SQL that does not parse: [`SQ0001`](/docs/diagnostics/reference/#sq0001), with the span where - the parser stopped. +- SQL that does not parse: [`SQ0001`](/docs/diagnostics/reference/#sq0001), at the point where the + parser stopped (or at a `(` that is never closed), with what it expected there. - Input beyond the parser's limits: the `SQ01xx` codes. - SQL that parses but that Snowflake rejects, such as `INTERSECT ALL`, `FOR UPDATE WAIT`, `QUALIFY` without a window function, or a `* ILIKE` `REPLACE` or `RENAME` of a column the pattern removes: the `SQ02xx` codes. +- With `--schema`, a relation the schema does not declare: + [`SQ0004`](/docs/diagnostics/reference/#sq0004); and a call to a function that is neither a + documented Snowflake function nor one the schema declares: + [`SQ0205`](/docs/diagnostics/reference/#sq0205). Without a schema, names are not checked. ## Options | Option | Meaning | |--------|---------| +| `--schema FILE` | A [schema file](/docs/schema/); report relations and functions it does not declare. | | `--format text\|json` | Print the report as text for people or JSON for programs. Default `text`. | | `--json` | The same as `--format json`. | diff --git a/website/src/content/docs/docs/diagnostics/reference.mdx b/website/src/content/docs/docs/diagnostics/reference.mdx index dd083367e..59d93b677 100644 --- a/website/src/content/docs/docs/diagnostics/reference.mdx +++ b/website/src/content/docs/docs/diagnostics/reference.mdx @@ -32,6 +32,7 @@ renumbered. See [Reading diagnostics](/docs/diagnostics/overview/) for the outpu | [`SQ0202`](#sq0202) | `lock_wait` | Snowflake does not support WAIT in FOR UPDATE | | [`SQ0203`](#sq0203) | `qualify_without_window` | QUALIFY needs a window function | | [`SQ0204`](#sq0204) | `star_filter_drops_named_column` | REPLACE or RENAME names a column the star's ILIKE pattern removed | +| [`SQ0205`](#sq0205) | `unknown_function` | Snowflake has no function of this name | ## Conversion @@ -306,3 +307,17 @@ In `* ILIKE 'pattern' REPLACE (...)` or `RENAME (...)`, Snowflake applies the pa ```sh echo "SELECT * ILIKE '%_id' REPLACE (price * 2 AS price) FROM orders" | sqlquack check ``` + +### SQ0205 + +**Snowflake has no function of this name** (`unknown_function`) + +The call names a function that is neither one of Snowflake's documented functions nor a function the schema declares, so Snowflake rejects the statement (Snowflake error 2140, unknown function). SQLQuack checks calls only when a schema is given, because a user-defined function can only be told apart from a misspelled one by the schema's `functions`. Calls qualified with a database or schema are not checked. + +**Help:** check the function's spelling, or declare a user-defined function under `functions` in the schema + +**Example:** + +```sh +echo "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders" | sqlquack check --schema schema.json +``` diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index bc16e23f9..86a786b0c 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -28,14 +28,24 @@ test('spans after non-ASCII text underline the characters the diagnostic names', }); test('a parse failure on a later line renders at its line and column', () => { - const sql = "SELECT '日本' AS note\nFROM orders\nWHERE"; + const sql = "SELECT '日本' AS note\nFROM orders\nWHERE x = = 1"; const report = JSON.parse(analyze(sql, SCHEMA)); assert.equal(report.parse.status, 'fail'); const [failure] = report.parse.diagnostics; assert.equal(failure.code, 'SQ0001'); - assert.equal(Array.from(sql).slice(failure.span.start, failure.span.end).join(''), 'WHERE'); - assert.match(rendered(failure, Array.from(sql)), /--> query\.sql:3:1\n/); - assert.deepEqual(marked(underlined(Array.from(sql), [{ kind: 'error', span: failure.span }]).html), ['WHERE']); + assert.equal(failure.label, 'unexpected `=`'); + assert.equal(Array.from(sql).slice(failure.span.start, failure.span.end).join(''), '='); + assert.match(rendered(failure, Array.from(sql)), /--> query\.sql:3:11\n/); + assert.deepEqual(marked(underlined(Array.from(sql), [{ kind: 'error', span: failure.span }]).html), ['=']); +}); + +test('with a schema, an unknown relation or function fails the checks', () => { + const checks = (sql) => JSON.parse(analyze(sql, SCHEMA)).checks; + assert.deepEqual(checks('SELECT order_id FROM orderz').findings.map((finding) => finding.code), ['E200']); + assert.deepEqual(checks('SELECT NOSUCHFUNCTION(order_id) AS x FROM orders').findings.map((finding) => finding.code), ['E202']); + const [quoted] = checks('SELECT order_id FROM orders WHERE note = "shipped"').findings; + assert.equal(quoted.help, "use single quotes for a string literal: 'shipped'"); + assert.equal(checks('SELECT order_id FROM orders').status, 'pass'); }); test('each stage reports pass, fail or skipped', () => { diff --git a/website/wasm/src/lib.rs b/website/wasm/src/lib.rs index f55df3877..e653b2c16 100644 --- a/website/wasm/src/lib.rs +++ b/website/wasm/src/lib.rs @@ -34,8 +34,7 @@ use sqlquack_rules::session::main::parse_session::parse_session; use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; use sqlquack_semantics::dialects::constants::SNOWFLAKE; use sqlquack_semantics::lineage::main::query_name_findings::query_name_findings; -use sqlquack_semantics::lineage::models::{LineageContext, Nullability, SchemaColumn}; -use sqlquack_semantics::lineage::types::LineageSchema; +use sqlquack_semantics::lineage::models::{LineageContext, SchemaCatalog}; use sqlquack_semantics::types::main::parse_schema::parse_schema; use sqlquack_semantics::types::models::Schema; use sqlquack_syntax::{Dialect, Expression, ParseOptions}; @@ -82,17 +81,14 @@ pub fn fixture_tables(schema: &str) -> String { json!({ "tables": tables }).to_string() } +/// The name check's code for a column no FROM item offers. +const UNKNOWN_COLUMN_FINDING: &str = "E201"; + /// The schema's relations as conversion types them, and as name checks resolve them. #[derive(Debug, Default)] struct Catalog { schema: Schema, - columns: BTreeMap>, -} - -impl LineageSchema for Catalog { - fn columns(&self, relation: &str) -> Option<&[SchemaColumn]> { - self.columns.get(relation).map(Vec::as_slice) - } + names: SchemaCatalog, } /// The schema file `text`, read by the same parser as `sqlquack convert --schema`. @@ -101,23 +97,8 @@ fn catalog(text: &str) -> Result { return Ok(Catalog::default()); } let schema = parse_schema(text).map_err(|error| error.to_string())?; - let columns = schema - .relations - .iter() - .map(|(relation, columns)| { - let named = columns - .iter() - .map(|column| SchemaColumn { - name: column.name.clone(), - quoted: false, - data_type: Some(column.data_type.to_string()), - nullability: Nullability::Unknown, - }) - .collect(); - (relation.clone(), named) - }) - .collect(); - Ok(Catalog { schema, columns }) + let names = SchemaCatalog::new(&schema); + Ok(Catalog { schema, names }) } /// The playground's policy and session settings. @@ -237,36 +218,43 @@ fn report(sql: &str, schema: &str, settings: &str) -> Value { .iter() .flat_map(|statement| snowflake_rejections(statement, (sql, &lines))) .collect(); - let findings: Vec = match (catalog.columns.is_empty(), statements.len()) { - (false, 1) => { + let names_checked = !catalog.schema.relations.is_empty() && statements.len() == 1; + let findings: Vec = match names_checked { + true => { let context = LineageContext { profile: &SNOWFLAKE, - schema: &catalog, + schema: &catalog.names, sql: Some(sql), }; query_name_findings(sql, Dialect::Snowflake, &ParseOptions::default(), context) .unwrap_or_default() .into_iter() .map(|finding| { + let written: Option = finding + .span + .map(|span| sql.chars().skip(span.start).take(span.end - span.start).collect()); + let help = written + .as_deref() + .and_then(|written| written.strip_prefix('"')?.strip_suffix('"')) + .filter(|_| finding.code == UNKNOWN_COLUMN_FINDING) + .map(|value| format!("use single quotes for a string literal: '{}'", value.replace('\'', "''"))); json!({ "code": finding.code, "message": finding.message, "span": finding.span.map(|span| json!({ "start": span.start, "end": span.end })), + "help": help, }) }) .collect() } - _ => Vec::new(), + false => Vec::new(), }; let mut checks = Stage::new( rejections.is_empty() && findings.is_empty(), diagnostics(&rejections, sql), ); checks.extra.insert("findings", json!(findings)); - checks.extra.insert( - "names_checked", - json!(!catalog.columns.is_empty() && statements.len() == 1), - ); + checks.extra.insert("names_checked", json!(names_checked)); // SQL Snowflake rejects is not translated, as the native check would refuse it at DuckDB's // bind, which the browser does not run. Warnings (W codes) do not stop translation. let erred = findings.iter().any(|finding| { From 4634d4948c47fb07f04210e7f46dd757b529dc14 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:45:19 +0000 Subject: [PATCH 08/12] Reject == in Snowflake SQL at parse, with a single = as the fix (D171) Snowflake rejects `status == 'shipped'` (2139) but the tokenizer read `==` as equality in every dialect, so SQLQuack translated it. Snowflake now tokenizes it as two `=`, which fails to parse with SQ0001 "unexpected `=`" and the help "Snowflake compares with a single `=`". Other dialects keep `==`. The everyday corpus query moves from translated to parse SQ0001. --- corpora/everyday/queries.toml | 4 ++-- .../src/diagnostics/_helpers/unparsed.rs | 20 ++++++++++++++++--- .../src/diagnostics/constants.rs | 4 ++++ .../src/diagnostics/tests/test_render.rs | 7 +++++++ crates/sqlquack-syntax/src/parser/models.rs | 1 + .../src/parser/tests/test_snowflake.rs | 6 ++++++ .../src/tokens/_helpers/scanning.rs | 2 +- crates/sqlquack-syntax/src/tokens/models.rs | 3 +++ .../src/tokens/tests/test_tokens.rs | 10 ++++++++++ .../analysis/tests/test_everyday_corpus.rs | 1 - 10 files changed, 51 insertions(+), 7 deletions(-) diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 572df7f7e..2ba7f9b2f 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -2986,8 +2986,8 @@ expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status == 'shipped'''' -expect = "translated" -expect_repository = "translated" +expect = "parse SQ0001" +expect_repository = "parse SQ0001" snowflake = "rejects 2139" [[query]] diff --git a/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs b/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs index 45546e298..2210f24bc 100644 --- a/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs +++ b/crates/sqlquack-core/src/diagnostics/_helpers/unparsed.rs @@ -4,8 +4,9 @@ use sqlquack_syntax::tokens::models::TokenType; use sqlquack_syntax::{Dialect, LosslessToken, TokenKind, tokenize}; use crate::diagnostics::constants::{ - CLOSING_PAREN, END_OF_INPUT, EXPECTED_PREFIX, FOUND_SEPARATOR, NEAR_SUFFIX, TEMPLATE_HELP, - TEMPLATE_MARKERS, TOKEN_TEXTS, UNCLOSED_PAREN_HELP, UNEXPECTED_TOKEN, + CLOSING_PAREN, DOUBLED_EQUALS, DOUBLED_EQUALS_HELP, END_OF_INPUT, EXPECTED_PREFIX, + FOUND_SEPARATOR, NEAR_SUFFIX, TEMPLATE_HELP, TEMPLATE_MARKERS, TOKEN_TEXTS, + UNCLOSED_PAREN_HELP, UNEXPECTED_TOKEN, }; use crate::parsing::models::{SourceLines, SourceLocation}; @@ -22,7 +23,20 @@ pub(crate) fn unparsed(message: &str, location: Option, source: let help = TEMPLATE_MARKERS .iter() .any(|marker| source.contains(marker)) - .then(|| TEMPLATE_HELP.to_owned()); + .then(|| TEMPLATE_HELP.to_owned()) + .or_else(|| { + location + .filter(|at| at.start > 0) + .is_some_and(|at| { + source + .chars() + .skip(at.start - 1) + .take(2) + .collect::() + == DOUBLED_EQUALS + }) + .then(|| DOUBLED_EQUALS_HELP.to_owned()) + }); let Some((expected, got)) = message .strip_prefix(EXPECTED_PREFIX) .and_then(|rest| rest.split_once(FOUND_SEPARATOR)) diff --git a/crates/sqlquack-core/src/diagnostics/constants.rs b/crates/sqlquack-core/src/diagnostics/constants.rs index ce60392a0..87b2f8cec 100644 --- a/crates/sqlquack-core/src/diagnostics/constants.rs +++ b/crates/sqlquack-core/src/diagnostics/constants.rs @@ -59,6 +59,10 @@ pub const UNCLOSED_PAREN_HELP: &str = "add the missing `)` where the bracketed p pub const TEMPLATE_MARKERS: [&str; 4] = ["{{", "}}", "{%", "%}"]; /// The help for SQL that still holds template markers. pub const TEMPLATE_HELP: &str = "render dbt/Jinja templates before checking"; +/// `==`, which Snowflake does not read as equality. +pub const DOUBLED_EQUALS: &str = "=="; +/// The fix for comparing with `==`. +pub const DOUBLED_EQUALS_HELP: &str = "Snowflake compares with a single `=`"; /// How an unresolved column's reason starts. pub const UNRESOLVED_COLUMN_PREFIX: &str = "column `"; /// The quote around a quoted identifier in Snowflake SQL. diff --git a/crates/sqlquack-core/src/diagnostics/tests/test_render.rs b/crates/sqlquack-core/src/diagnostics/tests/test_render.rs index 4f622b7a2..aa8fee009 100644 --- a/crates/sqlquack-core/src/diagnostics/tests/test_render.rs +++ b/crates/sqlquack-core/src/diagnostics/tests/test_render.rs @@ -182,6 +182,13 @@ fn given_everyday_parse_failures_when_diagnosed_then_the_label_names_what_was_ex expected_column: 27, expected_help: None, }, + UnparsedTestCase { + description: "== is pointed at its second `=`, with Snowflake's single `=` as the fix", + source: "SELECT order_id FROM orders WHERE status == 'shipped'", + expected_label: "unexpected `=`", + expected_column: 43, + expected_help: Some("Snowflake compares with a single `=`"), + }, UnparsedTestCase { description: "a dbt model is rendered before it is checked", source: "select\n {{ dbt_utils.generate_surrogate_key(['order_id', 'line_number']) }} as order_item_key,\n order_id\nfrom {{ ref('stg_order_items') }}", diff --git a/crates/sqlquack-syntax/src/parser/models.rs b/crates/sqlquack-syntax/src/parser/models.rs index 7a0d4936f..dc4c81dda 100644 --- a/crates/sqlquack-syntax/src/parser/models.rs +++ b/crates/sqlquack-syntax/src/parser/models.rs @@ -256,6 +256,7 @@ impl DialectImpl for SnowflakeDialect { ]; config.nested_comments = false; config.comments.insert("//".to_string(), None); + config.double_equals_is_eq = false; config } } diff --git a/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs b/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs index c6ac8e30f..74e781338 100644 --- a/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs +++ b/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs @@ -444,6 +444,12 @@ fn given_try_to_number_with_numbers_when_parsing_snowflake_then_they_are_precisi #[test] fn given_unsupported_snowflake_sql_when_parsing_then_parsing_fails() { let test_cases = [ + ParseErrorTestCase { + description: "== is not an equality operator", + dialect: Some(DialectType::Snowflake), + sql: "SELECT order_id FROM orders WHERE status == 'shipped'", + expected_error: "Parse error", + }, ParseErrorTestCase { description: "two commas after a projection are rejected", dialect: Some(DialectType::Snowflake), diff --git a/crates/sqlquack-syntax/src/tokens/_helpers/scanning.rs b/crates/sqlquack-syntax/src/tokens/_helpers/scanning.rs index 5b9cb04f1..b6a3751e6 100644 --- a/crates/sqlquack-syntax/src/tokens/_helpers/scanning.rs +++ b/crates/sqlquack-syntax/src/tokens/_helpers/scanning.rs @@ -828,7 +828,7 @@ impl<'a, C: TokenizerCursor, T: TokenOutput> TokenizerState<'a, C, T> { let token_type = match (c, next) { ('.', ':') => Some(TokenType::DotColon), - ('=', '=') => Some(TokenType::Eq), + ('=', '=') if self.config.double_equals_is_eq => Some(TokenType::Eq), ('<', '=') => Some(TokenType::Lte), ('>', '=') => Some(TokenType::Gte), ('!', '=') => Some(TokenType::Neq), diff --git a/crates/sqlquack-syntax/src/tokens/models.rs b/crates/sqlquack-syntax/src/tokens/models.rs index 89678c385..aaf4fdfa2 100644 --- a/crates/sqlquack-syntax/src/tokens/models.rs +++ b/crates/sqlquack-syntax/src/tokens/models.rs @@ -1404,6 +1404,8 @@ pub struct TokenizerConfig { pub recover_unterminated_string: bool, /// Whether `//` is the integer-division operator (Vertica). pub double_slash_int_div: bool, + /// Whether `==` is one equality operator; Snowflake reads it as two `=` and rejects it. + pub double_equals_is_eq: bool, } impl Default for TokenizerConfig { @@ -1432,6 +1434,7 @@ impl Default for TokenizerConfig { recover_terminal_backslash_quote: false, recover_unterminated_string: false, double_slash_int_div: false, + double_equals_is_eq: true, } } } diff --git a/crates/sqlquack-syntax/src/tokens/tests/test_tokens.rs b/crates/sqlquack-syntax/src/tokens/tests/test_tokens.rs index f20a86dd5..8afed78dc 100644 --- a/crates/sqlquack-syntax/src/tokens/tests/test_tokens.rs +++ b/crates/sqlquack-syntax/src/tokens/tests/test_tokens.rs @@ -100,6 +100,16 @@ fn given_sql_when_tokenizing_then_each_token_has_its_type_and_text() { (TokenType::Number, "3"), ]), }, + TokenizeTestCase { + description: "== is one equality token by default", + config: TokenizerConfig::default(), + sql: "a == b", + expected_tokens: Ok(vec![ + (TokenType::Var, "a"), + (TokenType::Eq, "=="), + (TokenType::Var, "b"), + ]), + }, TokenizeTestCase { description: "comparison operators", config: TokenizerConfig::default(), diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index b05d29a94..05cf5588f 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -15,7 +15,6 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i expected_rejections_not_caught: vec![ "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", - "SELECT order_id FROM orders WHERE status == 'shipped'", "SELECT order_id FROM orders WHERE status IN 'shipped'", "SELECT order_id FROM orders WHERE total >> 10", "SELECT order_id FROM orders WHERE ordered_at > 5", From 1ec3b1535a61819e33b3a8a59b1b61b0cb8c2c8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:58:36 +0000 Subject: [PATCH 09/12] Refuse IN before a bare value, >> and << in Snowflake SQL at parse (D171) Snowflake rejects `status IN 'shipped'`, `total >> 10` and `1 << 2` with syntax error 1003. In Snowflake's faithful modes IN and NOT IN now need `(` ("expected `(`, found `'shipped'`"), and >> and << are no shift operators ("unexpected `>>`"). `&`, `|` and `^` still parse: Snowflake parses them and fails at compile (2139, 2). Scripting's FOR ... IN is parsed elsewhere and is unchanged: none of the 56,748 receipted statements changes verdict. The == cases move to the faithful-mode tests with the new ones, and the two everyday corpus queries move to parse SQ0001. --- corpora/everyday/queries.toml | 8 +-- .../_helpers/expressions/comparisons.rs | 10 +++- .../src/parser/tests/test_snowflake.rs | 6 -- .../tests/parser/test_snowflake_fixes.rs | 56 +++++++++++++++++++ .../analysis/tests/test_everyday_corpus.rs | 2 - 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 2ba7f9b2f..1bfd14210 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -3017,8 +3017,8 @@ expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE status IN 'shipped'''' -expect = "refused SQ0002 node:in" -expect_repository = "refused SQ0002 node:in" +expect = "parse SQ0001" +expect_repository = "parse SQ0001" snowflake = "rejects 1003" [[query]] @@ -3091,8 +3091,8 @@ expect_repository = "parse SQ0001" [[query]] category = "invalid_syntax" sql = '''SELECT order_id FROM orders WHERE total >> 10''' -expect = "refused SQ0002 node:bitwise_right_shift" -expect_repository = "refused SQ0002 node:bitwise_right_shift" +expect = "parse SQ0001" +expect_repository = "parse SQ0001" snowflake = "rejects 1003" [[query]] diff --git a/crates/sqlquack-syntax/src/parser/_helpers/expressions/comparisons.rs b/crates/sqlquack-syntax/src/parser/_helpers/expressions/comparisons.rs index 72018d73b..e30bb8b67 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/expressions/comparisons.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/expressions/comparisons.rs @@ -609,6 +609,9 @@ impl Parser { } fn parse_in_after_not(&mut self, global_in: bool, left: Expression) -> Result { + if self.applies_snowflake_context_rules() && !self.check(TokenType::LParen) { + self.expect_token(TokenType::LParen)?; + } Ok({ if self.check_identifier("UNNEST") { self.skip(); @@ -769,6 +772,9 @@ impl Parser { } fn parse_in_comparison(&mut self, global_in: bool, left: Expression) -> Result { + if self.applies_snowflake_context_rules() && !self.check(TokenType::LParen) { + self.expect_token(TokenType::LParen)?; + } Ok({ if self.check_identifier("UNNEST") { self.skip(); @@ -1213,7 +1219,7 @@ impl Parser { expr_start: usize, ) -> Result { loop { - if self.match_token(TokenType::LtLt) { + if !self.applies_snowflake_context_rules() && self.match_token(TokenType::LtLt) { if self.config.dialect == Some(crate::tokens::models::DialectType::PostgreSQL) && self.match_token(TokenType::DArrow) { @@ -1225,7 +1231,7 @@ impl Parser { } let right = self.parse_duckdb_concat()?; left = Expression::BitwiseLeftShift(Box::new(BinaryOp::new(left, right))); - } else if self.match_token(TokenType::GtGt) { + } else if !self.applies_snowflake_context_rules() && self.match_token(TokenType::GtGt) { let right = self.parse_duckdb_concat()?; left = Expression::BitwiseRightShift(Box::new(BinaryOp::new(left, right))); } else { diff --git a/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs b/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs index 74e781338..c6ac8e30f 100644 --- a/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs +++ b/crates/sqlquack-syntax/src/parser/tests/test_snowflake.rs @@ -444,12 +444,6 @@ fn given_try_to_number_with_numbers_when_parsing_snowflake_then_they_are_precisi #[test] fn given_unsupported_snowflake_sql_when_parsing_then_parsing_fails() { let test_cases = [ - ParseErrorTestCase { - description: "== is not an equality operator", - dialect: Some(DialectType::Snowflake), - sql: "SELECT order_id FROM orders WHERE status == 'shipped'", - expected_error: "Parse error", - }, ParseErrorTestCase { description: "two commas after a projection are rejected", dialect: Some(DialectType::Snowflake), diff --git a/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs b/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs index f75f89d38..4046eb1ff 100644 --- a/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs +++ b/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs @@ -1253,3 +1253,59 @@ fn given_snowflake_secret_resource_monitor_and_integration_alters_when_parsing_t ); } } + +#[test] +fn given_an_operator_snowflake_does_not_have_when_parsing_then_it_is_refused() { + let test_cases = [ + SnowflakeVerdictTestCase { + description: "== for equality (recorded errno 2139, refused at parse per D171)", + sql: "SELECT order_id FROM orders WHERE status == 'shipped'", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: ">> as a shift (recorded errno 1003)", + sql: "SELECT order_id FROM orders WHERE total >> 10", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: "<< as a shift (recorded errno 1003)", + sql: "SELECT 1 << 2 ;", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: + "keeps & | and ^, which parse and fail at compile (recorded errnos 2139 and 2)", + sql: "SELECT 1 & 3 , 1 | 2 , 1 ^ 2 ;", + expected_kind: Some("select"), + }, + SnowflakeVerdictTestCase { + description: "IN before a bare value (recorded errno 1003)", + sql: "SELECT order_id FROM orders WHERE status IN 'shipped'", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: "NOT IN before a bare value", + sql: "SELECT order_id FROM orders WHERE status NOT IN 'shipped'", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: "keeps a single = and IN or NOT IN before a list or a query", + sql: "SELECT 1 FROM t WHERE a = 1 AND a IN ( 1 , 2 ) AND a NOT IN ( SELECT 1 ) ;", + expected_kind: Some("select"), + }, + SnowflakeVerdictTestCase { + description: "keeps > and >=", + sql: "SELECT 1 FROM t WHERE a > 1 AND a >= 2 ;", + expected_kind: Some("select"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + snowflake_kind(test_case.sql), + test_case.expected_kind.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index 05cf5588f..9d23c6e72 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -15,8 +15,6 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i expected_rejections_not_caught: vec![ "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", - "SELECT order_id FROM orders WHERE status IN 'shipped'", - "SELECT order_id FROM orders WHERE total >> 10", "SELECT order_id FROM orders WHERE ordered_at > 5", "SELECT LENGTH(order_id, 2) AS x FROM orders", "SELECT order_id FROM orders o, orders o", From a454163f04d2b47916a6a7d9172311893b7b21a7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:03:36 +0000 Subject: [PATCH 10/12] Refuse an aggregate FILTER clause in Snowflake SQL at parse Snowflake rejects `COUNT(*) FILTER (WHERE ...)` with syntax error 1003 at the `(`. In Snowflake's faithful modes no call takes a FILTER clause any more, so the error points there ("unexpected `(`"); the FILTER function over an array is unchanged, and no receipted statement changes verdict. The everyday corpus query moves to parse SQ0001. --- corpora/everyday/queries.toml | 4 ++-- .../src/parser/_helpers/functions/generic.rs | 5 +++-- .../tests/parser/test_snowflake_fixes.rs | 12 +++++++++++- .../src/analysis/tests/test_everyday_corpus.rs | 1 - 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index 1bfd14210..ca61e797f 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -751,8 +751,8 @@ expect_repository = "translated" [[query]] category = "aggregates" sql = '''SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders''' -expect = "refused SQ0002 fn:COUNT" -expect_repository = "refused SQ0002 fn:COUNT" +expect = "parse SQ0001" +expect_repository = "parse SQ0001" snowflake = "rejects 1003" [[query]] diff --git a/crates/sqlquack-syntax/src/parser/_helpers/functions/generic.rs b/crates/sqlquack-syntax/src/parser/_helpers/functions/generic.rs index 0a061e367..182d72d44 100644 --- a/crates/sqlquack-syntax/src/parser/_helpers/functions/generic.rs +++ b/crates/sqlquack-syntax/src/parser/_helpers/functions/generic.rs @@ -530,7 +530,7 @@ impl Parser { /// Parse optional FILTER clause pub(crate) fn parse_filter_clause(&mut self) -> Result> { - if self.match_token(TokenType::Filter) { + if !self.applies_snowflake_context_rules() && self.match_token(TokenType::Filter) { self.expect_token(TokenType::LParen)?; self.match_token(TokenType::Where); let filter_expr = self.parse_expression()?; @@ -595,7 +595,8 @@ impl Parser { expr }; - let expr = if self.match_token(TokenType::Filter) { + let expr = if !self.applies_snowflake_context_rules() && self.match_token(TokenType::Filter) + { self.expect_token(TokenType::LParen)?; self.expect_token(TokenType::Where)?; let filter_expr = self.parse_expression()?; diff --git a/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs b/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs index 4046eb1ff..4f125e039 100644 --- a/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs +++ b/crates/sqlquack-syntax/tests/parser/test_snowflake_fixes.rs @@ -1255,7 +1255,7 @@ fn given_snowflake_secret_resource_monitor_and_integration_alters_when_parsing_t } #[test] -fn given_an_operator_snowflake_does_not_have_when_parsing_then_it_is_refused() { +fn given_an_operator_or_clause_snowflake_does_not_have_when_parsing_then_it_is_refused() { let test_cases = [ SnowflakeVerdictTestCase { description: "== for equality (recorded errno 2139, refused at parse per D171)", @@ -1288,6 +1288,16 @@ fn given_an_operator_snowflake_does_not_have_when_parsing_then_it_is_refused() { sql: "SELECT order_id FROM orders WHERE status NOT IN 'shipped'", expected_kind: None, }, + SnowflakeVerdictTestCase { + description: "an aggregate FILTER clause (recorded errno 1003)", + sql: "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", + expected_kind: None, + }, + SnowflakeVerdictTestCase { + description: "keeps the FILTER function over an array", + sql: "SELECT FILTER ( [ 1 , 2 ] , x -> x > 1 ) ;", + expected_kind: Some("select"), + }, SnowflakeVerdictTestCase { description: "keeps a single = and IN or NOT IN before a list or a query", sql: "SELECT 1 FROM t WHERE a = 1 AND a IN ( 1 , 2 ) AND a NOT IN ( SELECT 1 ) ;", diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index 9d23c6e72..5b12f6a2d 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -13,7 +13,6 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i // SQL Snowflake rejects (snowflake-receipts.json) that SQLQuack neither fails to parse nor // flags in its checks; each should become a parse or check error. expected_rejections_not_caught: vec![ - "SELECT COUNT(*) FILTER (WHERE status = 'shipped') AS shipped FROM orders", "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", "SELECT order_id FROM orders WHERE ordered_at > 5", "SELECT LENGTH(order_id, 2) AS x FROM orders", From 075a7c235d67c7828f406b15cc12f67b07b9ed20 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:04:51 +0000 Subject: [PATCH 11/12] Check duplicate aliases, too many arguments and columns a PIVOT consumed (D171) Three queries Snowflake rejects without a schema now fail `check` instead of reaching translation, as structural Snowflake rejections that also stop the playground: - SQ0206 duplicate_alias: two FROM items or joins under one alias (Snowflake 2027), compared as Snowflake stores names, at the second alias. - SQ0207 too_many_arguments: a call passing more arguments than every catalogue signature allows (939). snowflake_function_arities.txt lists the 772 functions whose call grammars all have a fixed number of plain arguments; a test derives it from the catalogue. Calls with named arguments, and too few arguments, are not checked: receipts show two documented minimums (ARRAYS_ZIP, FORMAT_STRING_TEMPLATE) are not enforced. - SQ0208 pivot_consumed_column: reading the column a lone PIVOT aggregates or pivots on (904), outside SELECT-list aliases and nested queries. None of the three fires on any of the 24,472 parsed receipted statements or the 4,243 statements of the translation cases. Four everyday corpus queries move to check SQ0206, SQ0207 and SQ0208; ordered_at > 5 still needs a schema-typed check and stays a translation refusal. --- Cargo.lock | 1 + corpora/everyday/queries.toml | 12 +- crates/sqlquack-core/Cargo.toml | 3 + .../sqlquack-core/src/diagnostics/codes.lock | 3 + .../src/diagnostics/constants.rs | 46 ++ .../src/refusals/_helpers/clause_location.rs | 16 +- .../src/refusals/_helpers/rejections.rs | 146 +++- .../sqlquack-core/src/refusals/constants.rs | 4 + .../src/refusals/main/snowflake_rejections.rs | 3 +- crates/sqlquack-core/src/refusals/mod.rs | 1 + .../refusals/snowflake_function_arities.txt | 772 ++++++++++++++++++ .../src/refusals/tests/helpers.rs | 97 +++ .../src/refusals/tests/test_refusals.rs | 69 +- .../src/refusals/tests/test_types.rs | 5 + .../analysis/tests/test_everyday_corpus.rs | 3 - crates/sqlquack/tests/cli/test_check.rs | 16 + .../docs/docs/diagnostics/reference.mdx | 45 + 17 files changed, 1226 insertions(+), 16 deletions(-) create mode 100644 crates/sqlquack-core/src/refusals/constants.rs create mode 100644 crates/sqlquack-core/src/refusals/snowflake_function_arities.txt diff --git a/Cargo.lock b/Cargo.lock index 1a2796009..8bdcb6f12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1380,6 +1380,7 @@ dependencies = [ "serde", "serde_json", "sqlquack-syntax", + "toml", ] [[package]] diff --git a/corpora/everyday/queries.toml b/corpora/everyday/queries.toml index ca61e797f..0bdaca95e 100644 --- a/corpora/everyday/queries.toml +++ b/corpora/everyday/queries.toml @@ -2759,8 +2759,8 @@ expect_repository = "translated" [[query]] category = "misc" sql = '''SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p''' -expect = "refused SQ0003 node:pivot" -expect_repository = "refused SQ0003 node:pivot" +expect = "check SQ0208" +expect_repository = "check SQ0208" snowflake = "rejects 904" [[query]] @@ -3238,8 +3238,8 @@ snowflake = "rejects 1038" [[query]] category = "invalid_semantics" sql = '''SELECT LENGTH(order_id, 2) AS x FROM orders''' -expect = "refused SQ0002 fn:LENGTH" -expect_repository = "refused SQ0002 fn:LENGTH" +expect = "check SQ0207" +expect_repository = "check SQ0207" snowflake = "rejects 939" [[query]] @@ -3269,6 +3269,6 @@ expect_repository = "refused SQ0004 statement" [[query]] category = "invalid_semantics" sql = '''SELECT order_id FROM orders o, orders o''' -expect = "refused SQ0004 statement" -expect_repository = "refused SQ0004 statement" +expect = "check SQ0206" +expect_repository = "check SQ0206" snowflake = "rejects 2027" diff --git a/crates/sqlquack-core/Cargo.toml b/crates/sqlquack-core/Cargo.toml index f3cdd1671..9d264c09e 100644 --- a/crates/sqlquack-core/Cargo.toml +++ b/crates/sqlquack-core/Cargo.toml @@ -14,5 +14,8 @@ serde.workspace = true serde_json.workspace = true sqlquack-syntax.workspace = true +[dev-dependencies] +toml.workspace = true + [lints] workspace = true diff --git a/crates/sqlquack-core/src/diagnostics/codes.lock b/crates/sqlquack-core/src/diagnostics/codes.lock index 689b6e750..cad5a3c43 100644 --- a/crates/sqlquack-core/src/diagnostics/codes.lock +++ b/crates/sqlquack-core/src/diagnostics/codes.lock @@ -20,3 +20,6 @@ SQ0202 lock_wait SQ0203 qualify_without_window SQ0204 star_filter_drops_named_column SQ0205 unknown_function +SQ0206 duplicate_alias +SQ0207 too_many_arguments +SQ0208 pivot_consumed_column diff --git a/crates/sqlquack-core/src/diagnostics/constants.rs b/crates/sqlquack-core/src/diagnostics/constants.rs index 87b2f8cec..c89625c65 100644 --- a/crates/sqlquack-core/src/diagnostics/constants.rs +++ b/crates/sqlquack-core/src/diagnostics/constants.rs @@ -82,6 +82,12 @@ pub const MISSING_RULE: &str = "SQ0002"; pub const TYPE_CONDITION: &str = "SQ0003"; /// The name of the code of a call to a function neither Snowflake nor the schema declares. pub const UNKNOWN_FUNCTION: &str = "unknown_function"; +/// The name of the code of two FROM items under one alias. +pub const DUPLICATE_ALIAS: &str = "duplicate_alias"; +/// The name of the code of a call with more arguments than its Snowflake function takes. +pub const TOO_MANY_ARGUMENTS: &str = "too_many_arguments"; +/// The name of the code of a read of a column a PIVOT consumed. +pub const PIVOT_CONSUMED_COLUMN: &str = "pivot_consumed_column"; /// A relation or column name does not resolve. pub const UNRESOLVED_NAME: &str = "SQ0004"; /// DuckDB cannot express the construct's meaning. @@ -348,4 +354,44 @@ pub const CODES: &[CodeEntry] = &[ checked.", example: r#"echo "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders" | sqlquack check --schema schema.json"#, }, + CodeEntry { + code: "SQ0206", + name: DUPLICATE_ALIAS, + alias: None, + title: "Two FROM items share one alias", + help: Some("give each FROM item its own alias"), + explanation: "Each table, subquery or call in one FROM clause and its joins needs a \ + distinct alias, so Snowflake rejects a query that gives two of them the same one \ + (Snowflake error 2027, duplicate alias). Unquoted aliases compare in upper case, as \ + Snowflake stores them. Only aliases written with the items are compared.", + example: r#"echo "SELECT order_id FROM orders o, orders o" | sqlquack check"#, + }, + CodeEntry { + code: "SQ0207", + name: TOO_MANY_ARGUMENTS, + alias: None, + title: "The call passes more arguments than the function takes", + help: Some("remove the extra arguments; Snowflake's documentation lists each signature"), + explanation: "The call passes more arguments than any documented signature of the \ + Snowflake function accepts, so Snowflake rejects the statement (Snowflake error 939, \ + too many arguments). The limits come from the syntax catalogue's call grammars, and \ + only functions whose every signature has a fixed number of plain arguments are \ + checked. Calls with named arguments, and calls with too few arguments, are not \ + checked.", + example: r#"echo "SELECT LENGTH(order_id, 2) AS x FROM orders" | sqlquack check"#, + }, + CodeEntry { + code: "SQ0208", + name: PIVOT_CONSUMED_COLUMN, + alias: None, + title: "The query reads a column its PIVOT consumed", + help: Some( + "read the pivoted value columns, or select the column in a subquery before pivoting", + ), + explanation: "PIVOT replaces the column it aggregates and the column after FOR with one \ + column per listed value, so a query that still reads either by name fails in \ + Snowflake (Snowflake error 904, invalid identifier). This is checked when the PIVOT \ + is the only item in FROM, for names the SELECT list does not define as aliases.", + example: r#"echo "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p" | sqlquack check"#, + }, ]; diff --git a/crates/sqlquack-core/src/refusals/_helpers/clause_location.rs b/crates/sqlquack-core/src/refusals/_helpers/clause_location.rs index 9f51c2a86..b74d6f355 100644 --- a/crates/sqlquack-core/src/refusals/_helpers/clause_location.rs +++ b/crates/sqlquack-core/src/refusals/_helpers/clause_location.rs @@ -1,9 +1,11 @@ -//! Where a rejected node's offending clause is: a set operator with ALL, or a QUALIFY clause. +//! Where a rejected node's offending part is: a set operator with ALL, QUALIFY, an alias or a column. use sqlquack_syntax::Expression; +use crate::diagnostics::constants::{DUPLICATE_ALIAS, PIVOT_CONSUMED_COLUMN}; use crate::parsing::main::node_location::node_location; use crate::parsing::models::{SourceLines, SourceLocation}; +use crate::refusals::_helpers::rejections::{duplicate_alias, pivot_consumed_column}; /// The keyword that starts a QUALIFY clause. const QUALIFY_KEYWORD: &str = "QUALIFY"; @@ -12,13 +14,21 @@ const SET_OPERATORS: &[&str] = &["INTERSECT", "EXCEPT", "MINUS"]; /// The keyword after a set operator that keeps duplicates. const ALL_KEYWORD: &str = "ALL"; -/// The offending clause of a rejected node: a set operation's keywords, or a whole QUALIFY clause. +/// The offending part of a node rejected as `name`: set operation keywords, QUALIFY, or an alias. pub(crate) fn clause_location( - node: &Expression, + (node, name): (&Expression, &str), text: &[char], lines: &SourceLines, ) -> Option { match node { + Expression::Select(select) if name == DUPLICATE_ALIAS => duplicate_alias(select)? + .span + .map(|span| lines.location(span.start, span.end)), + Expression::Select(select) if name == PIVOT_CONSUMED_COLUMN => { + pivot_consumed_column(select)? + .span + .map(|span| lines.location(span.start, span.end)) + } Expression::Intersect(operation) => { operator_location(node, (&operation.left, &operation.right), (text, lines)) } diff --git a/crates/sqlquack-core/src/refusals/_helpers/rejections.rs b/crates/sqlquack-core/src/refusals/_helpers/rejections.rs index d45f7486d..d1703de64 100644 --- a/crates/sqlquack-core/src/refusals/_helpers/rejections.rs +++ b/crates/sqlquack-core/src/refusals/_helpers/rejections.rs @@ -1,8 +1,15 @@ //! The constructs Snowflake parses but then rejects, recognised one node at a time. -use sqlquack_syntax::expressions::names::models::Star; +use std::collections::HashSet; + +use sqlquack_syntax::expressions::functions::models::Function; +use sqlquack_syntax::expressions::names::models::{Column, Identifier, Star}; +use sqlquack_syntax::expressions::queries::models::Select; use sqlquack_syntax::{Expression, ExpressionWalk}; +use crate::diagnostics::constants::{DUPLICATE_ALIAS, PIVOT_CONSUMED_COLUMN, TOO_MANY_ARGUMENTS}; +use crate::refusals::constants::SNOWFLAKE_FUNCTION_ARITIES; + /// The registry name of the rejection a node causes, with the reason, if it causes one. pub(crate) fn rejection(node: &Expression) -> Option<(&'static str, String)> { match node { @@ -27,6 +34,36 @@ pub(crate) fn rejection(node: &Expression) -> Option<(&'static str, String)> { .to_owned(), )) } + Expression::Select(select) => duplicate_alias(select) + .map(|alias| { + ( + DUPLICATE_ALIAS, + format!("the alias `{}` names two FROM items", alias.name), + ) + }) + .or_else(|| { + pivot_consumed_column(select).map(|column| { + ( + PIVOT_CONSUMED_COLUMN, + format!( + "`{}` is aggregated or pivoted by the PIVOT, so its output has no such column", + column.name.name + ), + ) + }) + }), + Expression::Function(function) => most_arguments(function) + .filter(|most| function.args.len() > *most) + .map(|most| { + ( + TOO_MANY_ARGUMENTS, + format!( + "the most arguments {} takes is {most}, and this call passes {}", + function.name.to_uppercase(), + function.args.len() + ), + ) + }), Expression::Star(star) => filtered_out_name(star).map(|name| { ( "star_filter_drops_named_column", @@ -37,6 +74,113 @@ pub(crate) fn rejection(node: &Expression) -> Option<(&'static str, String)> { } } +/// The first alias written again on a later FROM item or join, compared as Snowflake stores it. +pub(crate) fn duplicate_alias(select: &Select) -> Option<&Identifier> { + let mut seen: HashSet = HashSet::new(); + let items = select.from.iter().flat_map(|from| from.expressions.iter()); + items + .chain(select.joins.iter().map(|join| &join.this)) + .filter_map(|item| match item { + Expression::Table(table) => table.alias.as_ref(), + Expression::Subquery(subquery) => subquery.alias.as_ref(), + Expression::Alias(alias) => Some(&alias.alias), + _ => None, + }) + .find(|alias| !seen.insert(stored_name(alias))) +} + +/// The first column a lone PIVOT aggregates or pivots on that its query still reads by that name. +pub(crate) fn pivot_consumed_column(select: &Select) -> Option<&Column> { + let [Expression::Pivot(pivot)] = select.from.as_ref()?.expressions.as_slice() else { + return None; + }; + let consumed: HashSet = pivot + .expressions + .iter() + .chain(&pivot.fields) + .flat_map(own_columns) + .map(|column| stored_name(&column.name)) + .collect(); + let aliases: HashSet = select + .expressions + .iter() + .filter_map(|projection| match projection { + Expression::Alias(alias) => Some(stored_name(&alias.alias)), + _ => None, + }) + .collect(); + let pivot_name = pivot.alias.as_ref().map(stored_name); + let clauses = select + .where_clause + .iter() + .map(|clause| &clause.this) + .chain(select.group_by.iter().flat_map(|group| &group.expressions)) + .chain(select.having.iter().map(|clause| &clause.this)) + .chain(select.qualify.iter().map(|clause| &clause.this)) + .chain( + select + .order_by + .iter() + .flat_map(|order| &order.expressions) + .map(|ordered| &ordered.this), + ); + select + .expressions + .iter() + .chain(clauses) + .flat_map(own_columns) + .filter(|_| select.joins.is_empty() && !pivot.unpivot) + .find(|column| { + let name = stored_name(&column.name); + consumed.contains(&name) + && !aliases.contains(&name) + && column + .table + .as_ref() + .is_none_or(|table| pivot_name.as_ref() == Some(&stored_name(table))) + }) +} + +/// The columns an expression reads outside any nested query. +fn own_columns(expression: &Expression) -> Vec<&Column> { + match expression { + Expression::Column(column) => vec![column], + Expression::Select(_) + | Expression::Subquery(_) + | Expression::Union(_) + | Expression::Intersect(_) + | Expression::Except(_) => Vec::new(), + _ => expression + .children() + .into_iter() + .flat_map(own_columns) + .collect(), + } +} + +/// An identifier as Snowflake stores it: as written when quoted, else in upper case. +fn stored_name(identifier: &Identifier) -> String { + match identifier.quoted { + true => identifier.name.clone(), + false => identifier.name.to_uppercase(), + } +} + +/// The most arguments a call without named arguments may pass, when the catalogue bounds it; a malformed limit bounds nothing. +fn most_arguments(function: &Function) -> Option { + let named = function + .args + .iter() + .any(|argument| matches!(argument, Expression::NamedArgument(_))); + let name = function.name.to_uppercase(); + SNOWFLAKE_FUNCTION_ARITIES + .lines() + .filter(|_| !named) + .filter_map(|line| line.split_once(' ')) + .find(|(listed, _)| *listed == name) + .map(|(_, most)| most.parse().unwrap_or(usize::MAX)) +} + /// A REPLACE or RENAME name the ILIKE pattern removes ignoring case; backslashes and non-ASCII skip. fn filtered_out_name(star: &Star) -> Option<&str> { let pattern = star diff --git a/crates/sqlquack-core/src/refusals/constants.rs b/crates/sqlquack-core/src/refusals/constants.rs new file mode 100644 index 000000000..d7903d28c --- /dev/null +++ b/crates/sqlquack-core/src/refusals/constants.rs @@ -0,0 +1,4 @@ +//! Constants for refusals. + +/// Each Snowflake function with a bounded argument count and its most arguments, one `NAME MAX` per line, from the syntax catalogue's call grammars. +pub(crate) const SNOWFLAKE_FUNCTION_ARITIES: &str = include_str!("snowflake_function_arities.txt"); diff --git a/crates/sqlquack-core/src/refusals/main/snowflake_rejections.rs b/crates/sqlquack-core/src/refusals/main/snowflake_rejections.rs index fcc28d4af..dfb4324eb 100644 --- a/crates/sqlquack-core/src/refusals/main/snowflake_rejections.rs +++ b/crates/sqlquack-core/src/refusals/main/snowflake_rejections.rs @@ -24,7 +24,8 @@ pub fn snowflake_rejections( let text = characters.get_or_init(|| source.chars().collect()); Some(Refusal { construct: Some(Construct::new(ConstructFamily::Syntax, name)), - location: clause_location(node, text, lines).or_else(|| node_location(node, lines)), + location: clause_location((node, name), text, lines) + .or_else(|| node_location(node, lines)), category: RefusalCategory::SnowflakeRejects, reason, }) diff --git a/crates/sqlquack-core/src/refusals/mod.rs b/crates/sqlquack-core/src/refusals/mod.rs index 7ade40c66..4671d991a 100644 --- a/crates/sqlquack-core/src/refusals/mod.rs +++ b/crates/sqlquack-core/src/refusals/mod.rs @@ -1,6 +1,7 @@ //! Structured refusals: what SQLQuack will not convert, where, and why. pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; pub mod types; diff --git a/crates/sqlquack-core/src/refusals/snowflake_function_arities.txt b/crates/sqlquack-core/src/refusals/snowflake_function_arities.txt new file mode 100644 index 000000000..2474cc3dc --- /dev/null +++ b/crates/sqlquack-core/src/refusals/snowflake_function_arities.txt @@ -0,0 +1,772 @@ +ABS 1 +ACCUMULATE 5 +ACOS 1 +ACOSH 1 +ADD_MONTHS 2 +AGG 1 +AI_AGG 2 +AI_CLASSIFY 4 +AI_COMPLETE 5 +AI_EMBED 2 +AI_GENERATE_TABLE_DESC 2 +AI_MULTI_EMBED 3 +AI_PARSE_DOCUMENT 3 +AI_REDACT 4 +AI_SENTIMENT 3 +AI_SIMILARITY 3 +AI_SUMMARIZE_AGG 1 +AI_TRANSCRIBE 3 +AI_TRANSLATE 4 +ALL_USER_NAMES 0 +APPROX_PERCENTILE_ACCUMULATE 1 +APPROX_PERCENTILE_COMBINE 1 +APPROX_PERCENTILE_ESTIMATE 2 +APPROX_TOP_K_ACCUMULATE 2 +APPROX_TOP_K_COMBINE 2 +APPROX_TOP_K_ESTIMATE 2 +ARRAYS_OVERLAP 2 +ARRAYS_TO_OBJECT 2 +ARRAY_APPEND 2 +ARRAY_CAT 2 +ARRAY_COMPACT 1 +ARRAY_CONTAINS 2 +ARRAY_DISTINCT 1 +ARRAY_EXCEPT 2 +ARRAY_FLATTEN 1 +ARRAY_GENERATE_RANGE 3 +ARRAY_INSERT 3 +ARRAY_INTERSECTION 2 +ARRAY_MAX 1 +ARRAY_MIN 1 +ARRAY_POSITION 2 +ARRAY_PREPEND 2 +ARRAY_REMOVE 2 +ARRAY_REMOVE_AT 2 +ARRAY_REPEAT 2 +ARRAY_REVERSE 1 +ARRAY_SIZE 1 +ARRAY_SLICE 3 +ARRAY_SORT 3 +ARRAY_SUM 1 +ARRAY_TO_STRING 2 +ASCII 1 +ASIN 1 +ASINH 1 +AS_ARRAY 1 +AS_BINARY 1 +AS_BOOLEAN 1 +AS_CHAR 1 +AS_DATE 1 +AS_DECIMAL 3 +AS_DOUBLE 1 +AS_INTEGER 1 +AS_NUMBER 3 +AS_OBJECT 1 +AS_REAL 1 +AS_TIME 1 +AS_TIMESTAMP_LTZ 1 +AS_TIMESTAMP_NTZ 1 +AS_TIMESTAMP_TZ 1 +AS_VARCHAR 1 +ATAN 1 +ATAN2 2 +ATANH 1 +BASE64_DECODE_BINARY 2 +BASE64_DECODE_STRING 2 +BASE64_ENCODE 3 +BIND_VALUES 1 +BITMAP_ABSOLUTE_POSITION 2 +BITMAP_AND 2 +BITMAP_AND_AGG 1 +BITMAP_BIT_POSITION 1 +BITMAP_BUCKET_NUMBER 1 +BITMAP_CONSTRUCT_AGG 1 +BITMAP_COUNT 1 +BITMAP_OR 2 +BITMAP_OR_AGG 1 +BITMAP_TO_ARRAY 2 +BITNOT 1 +BITSHIFTLEFT 2 +BITSHIFTRIGHT 2 +BIT_LENGTH 1 +BOOLAND 2 +BOOLNOT 1 +BOOLOR 2 +BOOLXOR 2 +CBRT 1 +CEIL 2 +CHARINDEX 3 +CHECK_JSON 1 +CHR 1 +COLLATION 1 +COMPRESS 2 +CONTAINS 2 +CONVERT_TIMEZONE 3 +COS 1 +COSH 1 +COT 1 +CURRENT_ACCOUNT 0 +CURRENT_ACCOUNT_NAME 0 +CURRENT_AVAILABLE_ROLES 0 +CURRENT_CLIENT 0 +CURRENT_DATABASE 0 +CURRENT_DATE 0 +CURRENT_IP_ADDRESS 0 +CURRENT_ORGANIZATION_NAME 0 +CURRENT_ORGANIZATION_USER 0 +CURRENT_REGION 0 +CURRENT_ROLE 0 +CURRENT_ROLE_TYPE 0 +CURRENT_SCHEMA 0 +CURRENT_SCHEMAS 0 +CURRENT_SECONDARY_ROLES 0 +CURRENT_SESSION 0 +CURRENT_STATEMENT 0 +CURRENT_TIME 1 +CURRENT_TIMESTAMP 1 +CURRENT_TRANSACTION 0 +CURRENT_USER 0 +CURRENT_VERSION 0 +CURRENT_WAREHOUSE 0 +DATASKETCHES_HLL_ESTIMATE 1 +DATEADD 3 +DATEDIFF 3 +DATE_FROM_PARTS 3 +DATE_TRUNC 2 +DAY 1 +DAYNAME 1 +DAYOFMONTH 1 +DAYOFWEEK 1 +DAYOFWEEKISO 1 +DAYOFYEAR 1 +DECOMPRESS_BINARY 2 +DECOMPRESS_STRING 2 +DECRYPT 4 +DECRYPT_RAW 6 +DEGREES 1 +DIV0 2 +DIV0NULL 2 +DP_INTERVAL_HIGH 1 +DP_INTERVAL_LOW 1 +EDITDISTANCE 3 +ENCRYPT 4 +ENCRYPT_RAW 5 +ENDSWITH 2 +EXECUTE_AI_EVALUATION 3 +EXP 1 +EXPLAIN_JSON 1 +FACTORIAL 1 +FILTER 2 +FLOOR 2 +FL_GET_CONTENT_TYPE 1 +FL_GET_ETAG 1 +FL_GET_FILE_TYPE 1 +FL_GET_LAST_MODIFIED 1 +FL_GET_RELATIVE_PATH 1 +FL_GET_SCOPED_FILE_URL 1 +FL_GET_SIZE 1 +FL_GET_STAGE 1 +FL_GET_STAGE_FILE_URL 1 +FL_IS_AUDIO 1 +FL_IS_COMPRESSED 1 +FL_IS_DOCUMENT 1 +FL_IS_IMAGE 1 +FL_IS_VIDEO 1 +GET 2 +GETBIT 2 +GETDATE 0 +GET_IGNORE_CASE 2 +GET_QUERY_OPERATOR_STATS 1 +H3_CELL_TO_BOUNDARY 1 +H3_CELL_TO_CHILDREN 2 +H3_CELL_TO_CHILDREN_STRING 2 +H3_CELL_TO_PARENT 2 +H3_CELL_TO_POINT 1 +H3_COMPACT_CELLS 1 +H3_COMPACT_CELLS_STRINGS 1 +H3_COVERAGE 2 +H3_COVERAGE_STRINGS 2 +H3_GET_RESOLUTION 1 +H3_GRID_DISK 2 +H3_GRID_DISTANCE 2 +H3_GRID_PATH 2 +H3_INT_TO_STRING 1 +H3_IS_PENTAGON 1 +H3_IS_VALID_CELL 1 +H3_LATLNG_TO_CELL 3 +H3_LATLNG_TO_CELL_STRING 3 +H3_POINT_TO_CELL 2 +H3_POINT_TO_CELL_STRING 2 +H3_POLYGON_TO_CELLS 2 +H3_POLYGON_TO_CELLS_STRINGS 2 +H3_STRING_TO_INT 1 +H3_TRY_COVERAGE 2 +H3_TRY_COVERAGE_STRINGS 2 +H3_TRY_GRID_DISTANCE 2 +H3_TRY_GRID_PATH 2 +H3_TRY_POLYGON_TO_CELLS 2 +H3_TRY_POLYGON_TO_CELLS_STRINGS 2 +H3_UNCOMPACT_CELLS 2 +H3_UNCOMPACT_CELLS_STRINGS 2 +HAVERSINE 4 +HEX_DECODE_BINARY 1 +HEX_DECODE_STRING 1 +HEX_ENCODE 2 +HOUR 1 +IFF 3 +IFNULL 2 +ILIKE 3 +INSERT 4 +INVOKER_ROLE 0 +INVOKER_SHARE 0 +IPADDRESS_TRUNC 2 +IS_ARRAY 1 +IS_BINARY 1 +IS_BOOLEAN 1 +IS_CHAR 1 +IS_DATE 1 +IS_DATE_VALUE 1 +IS_DECIMAL 1 +IS_DOUBLE 1 +IS_INTEGER 1 +IS_NULL_VALUE 1 +IS_OBJECT 1 +IS_ORGANIZATION_USER 1 +IS_REAL 1 +IS_TIME 1 +IS_TIMESTAMP_LTZ 1 +IS_TIMESTAMP_NTZ 1 +IS_TIMESTAMP_TZ 1 +IS_VARCHAR 1 +JAROWINKLER_SIMILARITY 2 +LAST_DAY 2 +LAST_QUERY_ID 1 +LAST_TRANSACTION 0 +LEFT 2 +LEN 1 +LENGTH 1 +LIKE 3 +LN 1 +LOCALTIME 0 +LOCALTIMESTAMP 1 +LOG 2 +LOWER 1 +LPAD 3 +LTRIM 2 +MAP_CAT 2 +MAP_CONTAINS_KEY 2 +MAP_ENTRIES 1 +MAP_INSERT 4 +MAP_KEYS 1 +MAP_SIZE 1 +MAX_BY 3 +MD5 1 +MD5_BINARY 1 +MD5_HEX 1 +MD5_NUMBER 1 +MD5_NUMBER_LOWER64 1 +MD5_NUMBER_UPPER64 1 +MINUTE 1 +MIN_BY 3 +MOD 2 +MONTH 1 +MONTHNAME 1 +MONTHS_BETWEEN 2 +NEXT_DAY 2 +NORMAL 3 +NULLIF 2 +NULLIFZERO 1 +NVL 2 +NVL2 3 +OBJECT_INSERT 4 +OBJECT_KEYS 1 +OCTET_LENGTH 1 +PARSE_URL 2 +PATTERN 1 +PERIOD_BEGIN 1 +PERIOD_CONSTRUCT 2 +PERIOD_CONTAINS 2 +PERIOD_END 1 +PERIOD_EQUALS 2 +PERIOD_IMMEDIATELY_PRECEDES 2 +PERIOD_IMMEDIATELY_SUCCEEDS 2 +PERIOD_INTERSECT 2 +PERIOD_LDIFF 2 +PERIOD_MEETS 2 +PERIOD_OVERLAPS 2 +PERIOD_PRECEDES 2 +PERIOD_RDIFF 2 +PERIOD_SUCCEEDS 2 +PI 0 +PREVIOUS_DAY 2 +QUARTER 1 +RADIANS 1 +RANDSTR 2 +REDUCE 3 +REGEXP_COUNT 4 +REGEXP_INSTR 7 +REGEXP_LIKE 3 +REGEXP_REPLACE 6 +REGEXP_SUBSTR 6 +REGEXP_SUBSTR_ALL 6 +REGR_VALX 2 +REGR_VALY 2 +REPEAT 2 +REPLACE 3 +REVERSE 1 +RIGHT 2 +RLIKE 3 +RPAD 3 +RTRIM 2 +RTRIMMED_LENGTH 1 +SECOND 1 +SHA1 1 +SHA1_BINARY 1 +SHA1_HEX 1 +SHA2 2 +SHA2_BINARY 2 +SHA2_HEX 2 +SIGN 1 +SIN 1 +SINH 1 +SKEW 1 +SNOWFLAKE.ALERT.GET_CONDITION_QUERY_UUID 0 +SNOWFLAKE.ALERT.LAST_SUCCESSFUL_SCHEDULED_TIME 0 +SNOWFLAKE.ALERT.SCHEDULED_TIME 0 +SNOWFLAKE.CORE.ACCURACY 1 +SNOWFLAKE.CORE.ALL_SPACE_COUNT 1 +SNOWFLAKE.CORE.ALL_SPACE_PERCENT 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_20 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_25 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_40 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_50 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_60 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_80 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_95 1 +SNOWFLAKE.CORE.APPROX_QUANTILE_99 1 +SNOWFLAKE.CORE.AVG 1 +SNOWFLAKE.CORE.BLANK_COUNT 1 +SNOWFLAKE.CORE.BLANK_PERCENT 1 +SNOWFLAKE.CORE.CASE_FORMAT_VIOLATION_COUNT 1 +SNOWFLAKE.CORE.CASE_FORMAT_VIOLATION_PERCENT 1 +SNOWFLAKE.CORE.CONTAINS_SPACES_COUNT 1 +SNOWFLAKE.CORE.CONTAINS_SPACES_PERCENT 1 +SNOWFLAKE.CORE.DATA_METRIC_SCHEDULED_TIME 0 +SNOWFLAKE.CORE.DUPLICATE_COUNT 1 +SNOWFLAKE.CORE.DUPLICATE_PERCENT 1 +SNOWFLAKE.CORE.EMPTY_STRING_COUNT 1 +SNOWFLAKE.CORE.EMPTY_STRING_PERCENT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_COUNT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_IQR_COUNT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_IQR_PERCENT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_PERCENT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_ZSCORE_COUNT 1 +SNOWFLAKE.CORE.EXTREME_OUTLIER_ZSCORE_PERCENT 1 +SNOWFLAKE.CORE.FALSE_COUNT 1 +SNOWFLAKE.CORE.FALSE_PERCENT 1 +SNOWFLAKE.CORE.FRESHNESS 1 +SNOWFLAKE.CORE.FUTURE_TIMESTAMP_COUNT 1 +SNOWFLAKE.CORE.FUTURE_TIMESTAMP_PERCENT 1 +SNOWFLAKE.CORE.GEOMETRIC_MEAN 1 +SNOWFLAKE.CORE.GET_PYTHON_PROFILER_OUTPUT 1 +SNOWFLAKE.CORE.HARMONIC_MEAN 1 +SNOWFLAKE.CORE.INVALID_CA_POSTAL_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_CA_POSTAL_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_CA_SIN_COUNT 1 +SNOWFLAKE.CORE.INVALID_CA_SIN_PERCENT 1 +SNOWFLAKE.CORE.INVALID_CH_OASI_COUNT 1 +SNOWFLAKE.CORE.INVALID_CH_OASI_PERCENT 1 +SNOWFLAKE.CORE.INVALID_CUSIP_COUNT 1 +SNOWFLAKE.CORE.INVALID_CUSIP_PERCENT 1 +SNOWFLAKE.CORE.INVALID_DE_POSTAL_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_DE_POSTAL_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_DE_TAX_ID_COUNT 1 +SNOWFLAKE.CORE.INVALID_DE_TAX_ID_PERCENT 1 +SNOWFLAKE.CORE.INVALID_EMAIL_COUNT 1 +SNOWFLAKE.CORE.INVALID_EMAIL_PERCENT 1 +SNOWFLAKE.CORE.INVALID_ES_DNI_COUNT 1 +SNOWFLAKE.CORE.INVALID_ES_DNI_PERCENT 1 +SNOWFLAKE.CORE.INVALID_FIGI_COUNT 1 +SNOWFLAKE.CORE.INVALID_FIGI_PERCENT 1 +SNOWFLAKE.CORE.INVALID_FR_INSEE_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_FR_INSEE_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_FR_POSTAL_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_FR_POSTAL_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_INTEGER_COUNT 1 +SNOWFLAKE.CORE.INVALID_INTEGER_PERCENT 1 +SNOWFLAKE.CORE.INVALID_ISIN_COUNT 1 +SNOWFLAKE.CORE.INVALID_ISIN_PERCENT 1 +SNOWFLAKE.CORE.INVALID_ISO8601_FORMAT_COUNT 1 +SNOWFLAKE.CORE.INVALID_ISO8601_FORMAT_PERCENT 1 +SNOWFLAKE.CORE.INVALID_IT_FISCAL_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_IT_FISCAL_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_JSON_COUNT 1 +SNOWFLAKE.CORE.INVALID_JSON_PERCENT 1 +SNOWFLAKE.CORE.INVALID_LATITUDE_COUNT 1 +SNOWFLAKE.CORE.INVALID_LATITUDE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_LEI_COUNT 1 +SNOWFLAKE.CORE.INVALID_LEI_PERCENT 1 +SNOWFLAKE.CORE.INVALID_LONGITUDE_COUNT 1 +SNOWFLAKE.CORE.INVALID_LONGITUDE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_NL_BSN_COUNT 1 +SNOWFLAKE.CORE.INVALID_NL_BSN_PERCENT 1 +SNOWFLAKE.CORE.INVALID_NUMERIC_TYPE_CAST_COUNT 1 +SNOWFLAKE.CORE.INVALID_NUMERIC_TYPE_CAST_PERCENT 1 +SNOWFLAKE.CORE.INVALID_PERCENTAGE_RANGE_COUNT 1 +SNOWFLAKE.CORE.INVALID_PERCENTAGE_RANGE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_PERM_ID_COUNT 1 +SNOWFLAKE.CORE.INVALID_PERM_ID_PERCENT 1 +SNOWFLAKE.CORE.INVALID_PL_PESEL_COUNT 1 +SNOWFLAKE.CORE.INVALID_PL_PESEL_PERCENT 1 +SNOWFLAKE.CORE.INVALID_PROBABILITY_RANGE_COUNT 1 +SNOWFLAKE.CORE.INVALID_PROBABILITY_RANGE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_SEDOL_COUNT 1 +SNOWFLAKE.CORE.INVALID_SEDOL_PERCENT 1 +SNOWFLAKE.CORE.INVALID_TIMESTAMP_STRING_COUNT 1 +SNOWFLAKE.CORE.INVALID_TIMESTAMP_STRING_PERCENT 1 +SNOWFLAKE.CORE.INVALID_TR_ID_NO_COUNT 1 +SNOWFLAKE.CORE.INVALID_TR_ID_NO_PERCENT 1 +SNOWFLAKE.CORE.INVALID_UK_NINO_COUNT 1 +SNOWFLAKE.CORE.INVALID_UK_NINO_PERCENT 1 +SNOWFLAKE.CORE.INVALID_UK_POSTAL_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_UK_POSTAL_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_USA_PHONE_COUNT 1 +SNOWFLAKE.CORE.INVALID_USA_PHONE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_USA_STATE_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_USA_STATE_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_USA_ZIP_CODE_COUNT 1 +SNOWFLAKE.CORE.INVALID_USA_ZIP_CODE_PERCENT 1 +SNOWFLAKE.CORE.INVALID_UUID_COUNT 1 +SNOWFLAKE.CORE.INVALID_UUID_PERCENT 1 +SNOWFLAKE.CORE.KURTOSIS 1 +SNOWFLAKE.CORE.MAE 1 +SNOWFLAKE.CORE.MAPE 1 +SNOWFLAKE.CORE.MAX 1 +SNOWFLAKE.CORE.MEAN_ERROR 1 +SNOWFLAKE.CORE.MEDIAN 1 +SNOWFLAKE.CORE.MIN 1 +SNOWFLAKE.CORE.NAN_COUNT 1 +SNOWFLAKE.CORE.NAN_PERCENT 1 +SNOWFLAKE.CORE.NEGATIVE_COUNT 1 +SNOWFLAKE.CORE.NEGATIVE_PERCENT 1 +SNOWFLAKE.CORE.NOT_FRIDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_FRIDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_IN_FUTURE_COUNT 1 +SNOWFLAKE.CORE.NOT_IN_FUTURE_PERCENT 1 +SNOWFLAKE.CORE.NOT_MONDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_MONDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_NULL_COUNT 1 +SNOWFLAKE.CORE.NOT_NULL_PERCENT 1 +SNOWFLAKE.CORE.NOT_SATURDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_SATURDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_SUNDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_SUNDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_THURSDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_THURSDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_TODAY_COUNT 1 +SNOWFLAKE.CORE.NOT_TODAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_TUESDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_TUESDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_WEDNESDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_WEDNESDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_WEEKDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_WEEKDAY_PERCENT 1 +SNOWFLAKE.CORE.NOT_YESTERDAY_COUNT 1 +SNOWFLAKE.CORE.NOT_YESTERDAY_PERCENT 1 +SNOWFLAKE.CORE.NULL_COUNT 1 +SNOWFLAKE.CORE.NULL_KEYWORD_COUNT 1 +SNOWFLAKE.CORE.NULL_KEYWORD_PERCENT 1 +SNOWFLAKE.CORE.NULL_PERCENT 1 +SNOWFLAKE.CORE.OUTLIER_COUNT 1 +SNOWFLAKE.CORE.OUTLIER_IQR_COUNT 1 +SNOWFLAKE.CORE.OUTLIER_IQR_PERCENT 1 +SNOWFLAKE.CORE.OUTLIER_PERCENT 1 +SNOWFLAKE.CORE.OUTLIER_ZSCORE_COUNT 1 +SNOWFLAKE.CORE.OUTLIER_ZSCORE_PERCENT 1 +SNOWFLAKE.CORE.RMSE 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_DAY_COUNT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_DAY_PERCENT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_HOUR_COUNT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_HOUR_PERCENT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_MONTH_COUNT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_MONTH_PERCENT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_WEEK_COUNT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_WEEK_PERCENT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_YEAR_COUNT 1 +SNOWFLAKE.CORE.ROWS_IN_PAST_YEAR_PERCENT 1 +SNOWFLAKE.CORE.R_SQUARED 1 +SNOWFLAKE.CORE.SKEW 1 +SNOWFLAKE.CORE.SPECIAL_CHARACTER_COUNT 1 +SNOWFLAKE.CORE.SPECIAL_CHARACTER_PERCENT 1 +SNOWFLAKE.CORE.STDDEV 1 +SNOWFLAKE.CORE.STRING_LENGTH_AVG 1 +SNOWFLAKE.CORE.STRING_LENGTH_MAX 1 +SNOWFLAKE.CORE.STRING_LENGTH_MIN 1 +SNOWFLAKE.CORE.STRING_LENGTH_STDDEV 1 +SNOWFLAKE.CORE.SUM 1 +SNOWFLAKE.CORE.TRUE_COUNT 1 +SNOWFLAKE.CORE.TRUE_PERCENT 1 +SNOWFLAKE.CORE.UNIQUE_COUNT 1 +SNOWFLAKE.CORE.UNIQUE_PERCENT 1 +SNOWFLAKE.CORE.UNIX_ZERO_COUNT 1 +SNOWFLAKE.CORE.UNIX_ZERO_PERCENT 1 +SNOWFLAKE.CORE.UNTRIMMED_STRING_COUNT 1 +SNOWFLAKE.CORE.UNTRIMMED_STRING_PERCENT 1 +SNOWFLAKE.CORE.VARIANCE 1 +SNOWFLAKE.CORE.ZERO_COUNT 1 +SNOWFLAKE.CORE.ZERO_PERCENT 1 +SNOWFLAKE.CORTEX.AGENT_RUN 2 +SNOWFLAKE.CORTEX.CLASSIFY_TEXT 3 +SNOWFLAKE.CORTEX.COUNT_TOKENS 2 +SNOWFLAKE.CORTEX.EMBED_TEXT_1024 2 +SNOWFLAKE.CORTEX.EMBED_TEXT_768 2 +SNOWFLAKE.CORTEX.ENTITY_SENTIMENT 2 +SNOWFLAKE.CORTEX.EXTRACT_ANSWER 2 +SNOWFLAKE.CORTEX.SENTIMENT 1 +SNOWFLAKE.CORTEX.THREAD_MESSAGES 3 +SNOWFLAKE.CORTEX.TRANSLATE 3 +SNOWFLAKE.CORTEX.TRY_COMPLETE 3 +SNOWFLAKE.DATA_PRIVACY.GENERATE_SYNTHETIC_DATA 1 +SNOWFLAKE.LOCAL.GET_AI_EVALUATION_DATA 5 +SNOWFLAKE.LOCAL.GET_AI_OBSERVABILITY_EVENTS 4 +SNOWFLAKE.LOCAL.GET_AI_OBSERVABILITY_LOGS 4 +SNOWFLAKE.LOCAL.GET_AI_RECORD_TRACE 5 +SNOWFLAKE.LOCAL.MY_BUDGET_SUMMARY 0 +SNOWFLAKE.NOTIFICATION.SANITIZE_WEBHOOK_CONTENT 1 +SNOWFLAKE.TELEMETRY.ADD_ROW_ACCESS_POLICY_ON_EVENTS_VIEW 2 +SNOWFLAKE.TELEMETRY.DROP_ROW_ACCESS_POLICY_ON_EVENTS_VIEW 1 +SOUNDEX 1 +SOUNDEX_P123 1 +SPACE 1 +SPLIT 2 +SPLIT_PART 3 +SPLIT_TO_TABLE 2 +STARTSWITH 2 +STRIP_NULL_VALUE 1 +STRTOK 3 +STRTOK_SPLIT_TO_TABLE 2 +STRTOK_TO_ARRAY 2 +ST_AREA 1 +ST_ASBINARY 1 +ST_ASEWKB 1 +ST_ASEWKT 1 +ST_ASGEOJSON 1 +ST_ASTEXT 1 +ST_ASWKB 1 +ST_ASWKT 1 +ST_AZIMUTH 2 +ST_BUFFER 2 +ST_CENTROID 1 +ST_COLLECT 2 +ST_CONTAINS 2 +ST_COVEREDBY 2 +ST_COVERS 2 +ST_DIFFERENCE 2 +ST_DIMENSION 1 +ST_DISJOINT 2 +ST_DISTANCE 2 +ST_DWITHIN 3 +ST_ENDPOINT 1 +ST_ENVELOPE 1 +ST_GEOGFROMEWKB 2 +ST_GEOGFROMEWKT 2 +ST_GEOGFROMGEOHASH 2 +ST_GEOGFROMTEXT 2 +ST_GEOGFROMWKB 2 +ST_GEOGFROMWKT 2 +ST_GEOGPOINTFROMGEOHASH 1 +ST_GEOGRAPHYFROMEWKB 2 +ST_GEOGRAPHYFROMEWKT 2 +ST_GEOGRAPHYFROMTEXT 2 +ST_GEOGRAPHYFROMWKB 2 +ST_GEOGRAPHYFROMWKT 2 +ST_GEOHASH 2 +ST_GEOMETRYFROMEWKB 3 +ST_GEOMETRYFROMEWKT 3 +ST_GEOMETRYFROMTEXT 3 +ST_GEOMETRYFROMWKB 3 +ST_GEOMETRYFROMWKT 3 +ST_GEOMFROMEWKB 3 +ST_GEOMFROMEWKT 3 +ST_GEOMFROMGEOHASH 2 +ST_GEOMFROMTEXT 3 +ST_GEOMFROMWKB 3 +ST_GEOMFROMWKT 3 +ST_GEOMPOINTFROMGEOHASH 1 +ST_HAUSDORFFDISTANCE 2 +ST_INTERPOLATE 2 +ST_INTERSECTION 2 +ST_INTERSECTION_AGG 1 +ST_INTERSECTS 2 +ST_ISVALID 1 +ST_LENGTH 1 +ST_MAKEGEOMPOINT 2 +ST_MAKELINE 2 +ST_MAKEPOINT 2 +ST_MAKEPOLYGON 1 +ST_MAKEPOLYGONORIENTED 1 +ST_NPOINTS 1 +ST_PERIMETER 1 +ST_POINTN 2 +ST_SETSRID 2 +ST_SIMPLIFY 3 +ST_SRID 1 +ST_STARTPOINT 1 +ST_SYMDIFFERENCE 2 +ST_UNION 2 +ST_UNION_AGG 1 +ST_WITHIN 2 +ST_X 1 +ST_XMAX 1 +ST_XMIN 1 +ST_Y 1 +ST_YMAX 1 +ST_YMIN 1 +SUBSTR 3 +SUBSTRING 3 +SYSDATE 0 +SYSTEM$ABORT_SESSION 1 +SYSTEM$ABORT_TRANSACTION 1 +SYSTEM$ACTIVATE_CMK_INFO 1 +SYSTEM$ACTIVATE_CMK_INFO_POSTGRES 0 +SYSTEM$ACTIVATE_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$ALLOWLIST 0 +SYSTEM$ALLOWLIST_PRIVATELINK 0 +SYSTEM$APP_COMPATIBILITY_CHECK 0 +SYSTEM$BLOCK_INTERNAL_STAGES_PUBLIC_ACCESS 0 +SYSTEM$BLOCK_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS 0 +SYSTEM$CANCEL_ALL_QUERIES 1 +SYSTEM$CANCEL_QUERY 1 +SYSTEM$CLIENT_VERSION_INFO 0 +SYSTEM$CLIENT_VULNERABILITY_INFO 0 +SYSTEM$COMMIT_MOVE_ORGANIZATION_ACCOUNT 1 +SYSTEM$CURRENT_USER_TASK_NAME 0 +SYSTEM$DEACTIVATE_CMK_INFO 0 +SYSTEM$DEACTIVATE_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$DEREGISTER_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$DISABLE_PREVIEW_ACCESS 0 +SYSTEM$DISABLE_PRIVATELINK_ACCESS_ONLY 0 +SYSTEM$ENABLE_PREVIEW_ACCESS 0 +SYSTEM$END_DEBUG_APPLICATION 0 +SYSTEM$ENFORCE_PRIVATELINK_ACCESS_ONLY 0 +SYSTEM$EXPLAIN_JSON_TO_TEXT 1 +SYSTEM$GET_ALERT_CONFIG 1 +SYSTEM$GET_ALL_DEFAULT_COLUMNS_OVERRIDES 0 +SYSTEM$GET_CMK_CONFIG_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$GET_CMK_INFO_POSTGRES 0 +SYSTEM$GET_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$GET_CMK_KMS_KEY_POLICY 0 +SYSTEM$GET_COMPUTE_POOL_PENDING_MAINTENANCE 0 +SYSTEM$GET_DEBUG_STATUS 0 +SYSTEM$GET_GCP_KMS_CMK_GRANT_ACCESS_CMD 0 +SYSTEM$GET_PREVIEW_ACCESS_STATUS 0 +SYSTEM$GET_PRIVATELINK_AUTHORIZED_ENDPOINTS 0 +SYSTEM$GET_PRIVATELINK_CONFIG 0 +SYSTEM$GET_PRIVATELINK_ENDPOINTS_INFO 0 +SYSTEM$GET_PRIVATELINK_ENDPOINT_REGISTRATIONS 0 +SYSTEM$GET_PURCHASE_ATTRIBUTES 0 +SYSTEM$GET_RESULTSET_STATUS 1 +SYSTEM$GET_SERVICE_DNS_DOMAIN 1 +SYSTEM$GET_SERVICE_LOGS 5 +SYSTEM$GET_SNOWFLAKE_EGRESS_IP_RANGES 1 +SYSTEM$GET_SNOWFLAKE_PLATFORM_INFO 0 +SYSTEM$GET_STAGE_PRIVATELINK_AUTHORIZED_ENDPOINTS 0 +SYSTEM$GET_TASK_GRAPH_CONFIG 1 +SYSTEM$GET_TZDB_VERSION 0 +SYSTEM$INTERNAL_STAGES_PUBLIC_ACCESS_STATUS 0 +SYSTEM$ISSUE_PER_ACCOUNT_APP_SERVICE_CERTIFICATE 0 +SYSTEM$ISSUE_PER_ACCOUNT_CERTIFICATES 0 +SYSTEM$IS_APPLICATION_AUTHORIZED_FOR_TELEMETRY_EVENT_SHARING 0 +SYSTEM$IS_APPLICATION_INSTALLED_FROM_SAME_ACCOUNT 0 +SYSTEM$IS_APPLICATION_SHARING_EVENTS_WITH_PROVIDER 0 +SYSTEM$IS_LISTING_PURCHASED 0 +SYSTEM$IS_LISTING_TRIAL 0 +SYSTEM$LINK_ORGANIZATION_USER_GROUP 1 +SYSTEM$LOG_DEBUG 1 +SYSTEM$LOG_ERROR 1 +SYSTEM$LOG_FATAL 1 +SYSTEM$LOG_INFO 1 +SYSTEM$LOG_TRACE 1 +SYSTEM$LOG_WARN 1 +SYSTEM$OPT_IN_INTERNAL_STAGE_NETWORK_LOGS 0 +SYSTEM$OPT_OUT_INTERNAL_STAGE_NETWORK_LOGS 0 +SYSTEM$SCHEDULE_ASYNC_REPLICATION_GROUP_REFRESH 1 +SYSTEM$SHOW_ACTIVE_BEHAVIOR_CHANGE_BUNDLES 0 +SYSTEM$SHOW_BUDGETS_IN_ACCOUNT 0 +SYSTEM$SHOW_EVENT_SHARING_ACCOUNTS 0 +SYSTEM$SHOW_MOVE_ORGANIZATION_ACCOUNT_STATUS 0 +SYSTEM$SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS_STATUS 0 +SYSTEM$SUPPORTED_DBT_VERSIONS 0 +SYSTEM$TYPEOF 1 +SYSTEM$UNBLOCK_INTERNAL_STAGES_PUBLIC_ACCESS 0 +SYSTEM$UNBLOCK_SNOWFLAKE_MANAGED_STORAGE_VOLUME_PUBLIC_ACCESS 0 +SYSTEM$VERIFY_CMK_INFO_POSTGRES 0 +SYSTEM$VERIFY_CMK_INFO_SNOWFLAKE_MANAGED_VOLUME 0 +SYSTEM$WHITELIST 0 +SYSTEM$WHITELIST_PRIVATELINK 0 +SYSTIMESTAMP 0 +TAN 1 +TANH 1 +TIMEADD 3 +TIMEDIFF 3 +TIMESTAMPADD 3 +TIMESTAMPDIFF 3 +TIMESTAMP_FROM_PARTS 8 +TIMESTAMP_LTZ_FROM_PARTS 7 +TIMESTAMP_NTZ_FROM_PARTS 7 +TIMESTAMP_TZ_FROM_PARTS 8 +TIME_FROM_PARTS 4 +TIME_SLICE 4 +TM9 2 +TO_ARRAY 1 +TO_BOOLEAN 1 +TO_FILE 2 +TO_GEOGRAPHY 2 +TO_GEOMETRY 3 +TO_IPADDRESS 1 +TO_JSON 1 +TO_OBJECT 1 +TO_UUID 1 +TO_VARIANT 1 +TO_XML 1 +TRANSFORM 2 +TRANSLATE 3 +TRIM 2 +TRUNC 2 +TRUNCATE 2 +TRY_BASE64_DECODE_BINARY 2 +TRY_BASE64_DECODE_STRING 2 +TRY_DECRYPT 4 +TRY_DECRYPT_RAW 6 +TRY_HEX_DECODE_BINARY 1 +TRY_HEX_DECODE_STRING 1 +TRY_TO_BOOLEAN 1 +TRY_TO_FILE 2 +TRY_TO_GEOGRAPHY 2 +TRY_TO_GEOMETRY 3 +TRY_TO_IPADDRESS 1 +TRY_TO_UUID 1 +TYPEOF 1 +UNICODE 1 +UNIFORM 3 +UPPER 1 +VECTOR 2 +VECTOR_AVG 1 +VECTOR_COSINE_SIMILARITY 2 +VECTOR_INNER_PRODUCT 2 +VECTOR_L1_DISTANCE 2 +VECTOR_L2_DISTANCE 2 +VECTOR_MAX 1 +VECTOR_MIN 1 +VECTOR_NORMALIZE 1 +VECTOR_SUM 1 +VECTOR_TRUNCATE 2 +WEEK 1 +WEEKISO 1 +WEEKOFYEAR 1 +WIDTH_BUCKET 4 +XMLGET 3 +YEAR 1 +YEAROFWEEK 1 +YEAROFWEEKISO 1 +ZEROIFNULL 1 diff --git a/crates/sqlquack-core/src/refusals/tests/helpers.rs b/crates/sqlquack-core/src/refusals/tests/helpers.rs index 3d5427cff..50a3d83ed 100644 --- a/crates/sqlquack-core/src/refusals/tests/helpers.rs +++ b/crates/sqlquack-core/src/refusals/tests/helpers.rs @@ -1,3 +1,6 @@ +use std::collections::BTreeMap; +use std::path::Path; + use crate::classification::main::classify::classify; use crate::classification::models::Occurrence; use crate::diagnostics::main::diagnose::diagnose; @@ -77,3 +80,97 @@ pub(crate) fn rejection_codes(sql: &str) -> Vec<&'static str> { .map(|diagnostic| diagnostic.code) .collect() } + +/// The `NAME MAX` lines of the functions every catalogue call grammar bounds, sorted by name. +pub(crate) fn catalogue_function_arities() -> String { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../coverage/syntax/catalogue"); + let grammars: Vec = std::fs::read_dir(root) + .expect("the catalogue reads") + .map(|entry| entry.expect("a catalogue file").path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "toml") + }) + .map(|path| std::fs::read_to_string(path).expect("a catalogue file reads")) + .map(|text| toml::from_str::(&text).expect("a catalogue file parses")) + .flat_map(|catalogue| array(catalogue.get("form"))) + .flat_map(|form| array(form.get("grammar"))) + .filter_map(|grammar| grammar.as_str().map(str::to_owned)) + .collect(); + let mut most: BTreeMap> = BTreeMap::new(); + for (name, arguments) in grammars.iter().filter_map(|grammar| call_parts(grammar)) { + let bound = arguments.and_then(most_in_grammar); + most.entry(name) + .and_modify(|known| *known = known.zip(bound).map(|(a, b)| a.max(b))) + .or_insert(bound); + } + most.into_iter() + .filter_map(|(name, bound)| bound.map(|bound| format!("{name} {bound}\n"))) + .collect() +} + +/// The entries of a TOML array, or none. +fn array(value: Option<&toml::Value>) -> Vec { + value + .and_then(toml::Value::as_array) + .cloned() + .unwrap_or_default() +} + +/// A call grammar's function name and its argument text, which is absent when the call is unclosed. +fn call_parts(grammar: &str) -> Option<(String, Option<&str>)> { + let (head, rest) = grammar.trim_start().split_once('(')?; + let name = head.trim_end(); + let mut characters = name.chars(); + let starts = characters + .next() + .is_some_and(|first| first.is_ascii_uppercase() || first == '_'); + let continues = characters.all(|character| { + character.is_ascii_uppercase() || character.is_ascii_digit() || "_.$".contains(character) + }); + (starts && continues).then(|| (name.to_owned(), rest.trim_end().strip_suffix(')'))) +} + +/// The most arguments a grammar of plain ``s, `[ , ... ]` options and commas takes. +fn most_in_grammar(arguments: &str) -> Option { + let spaced = grammar_spaced(arguments); + let tokens: Vec<&str> = spaced.split_whitespace().collect(); + let known = tokens.iter().all(|token| GRAMMAR_TOKENS.contains(token)); + let balanced = tokens.iter().try_fold(0_usize, |depth, token| { + (depth + usize::from(*token == "[")).checked_sub(usize::from(*token == "]")) + }) == Some(0); + let plain: Vec<&str> = tokens + .iter() + .copied() + .filter(|token| *token != "[" && *token != "]") + .collect(); + let alternating = plain.is_empty() + || plain.first() == Some(&ARGUMENT) + && plain.last() == Some(&ARGUMENT) + && plain.windows(2).all(|pair| pair[0] != pair[1]); + let count = plain.iter().filter(|token| **token == ARGUMENT).count(); + (known && balanced && alternating).then_some(count) +} + +/// A grammar's arguments as `<>`, and its brackets and commas, each set apart by spaces. +fn grammar_spaced(text: &str) -> String { + let mut parts = text.split('<'); + let first = parts.next().unwrap_or_default().replace('>', " "); + let arguments: String = parts + .map(|part| { + part.split_once('>').map_or_else( + || format!(" <{}", part.replace('>', " ")), + |(_, rest)| format!(" {ARGUMENT} {}", rest.replace('>', " ")), + ) + }) + .collect(); + (first + &arguments) + .replace('[', " [ ") + .replace(']', " ] ") + .replace(',', " , ") +} + +/// How [`grammar_spaced`] writes an argument. +const ARGUMENT: &str = "<>"; +/// The tokens a grammar of plain arguments holds. +const GRAMMAR_TOKENS: [&str; 4] = [ARGUMENT, "[", "]", ","]; diff --git a/crates/sqlquack-core/src/refusals/tests/test_refusals.rs b/crates/sqlquack-core/src/refusals/tests/test_refusals.rs index 2ac00ac8f..8002c2f92 100644 --- a/crates/sqlquack-core/src/refusals/tests/test_refusals.rs +++ b/crates/sqlquack-core/src/refusals/tests/test_refusals.rs @@ -1,9 +1,11 @@ +use crate::refusals::constants::SNOWFLAKE_FUNCTION_ARITIES; use crate::refusals::models::RefusalCategory; use crate::refusals::tests::helpers::{ - not_expressible_reasons, rejection_codes, rejection_names, unmatched_refusals, unparsed_refusal, + catalogue_function_arities, not_expressible_reasons, rejection_codes, rejection_names, + unmatched_refusals, unparsed_refusal, }; use crate::refusals::tests::test_types::{ - ConstructRefusalTestCase, RejectionTestCase, UnparsedTestCase, + ArityListTestCase, ConstructRefusalTestCase, RejectionTestCase, UnparsedTestCase, }; #[test] @@ -114,6 +116,53 @@ fn given_sql_snowflake_parses_then_rejects_when_refusing_then_each_rejection_is_ ], expected_codes: vec!["SQ0204", "SQ0204"], }, + RejectionTestCase { + description: "two FROM items under one alias are rejected, a quoted upper-case one too", + sql: "SELECT order_id FROM orders o, orders o; \ + SELECT 1 FROM orders o JOIN (SELECT 1 AS a) \"O\" ON TRUE", + expected_rejections: vec!["duplicate_alias", "duplicate_alias"], + expected_codes: vec!["SQ0206", "SQ0206"], + }, + RejectionTestCase { + description: "distinct aliases, a quoted lower-case one, unaliased items and nested scopes are not", + sql: "SELECT 1 FROM orders o, orders p; \ + SELECT 1 FROM orders o JOIN orders \"o\" ON TRUE; \ + SELECT 1 FROM orders JOIN orders ON TRUE; \ + SELECT 1 FROM orders o WHERE EXISTS (SELECT 1 FROM orders o)", + expected_rejections: vec![], + expected_codes: vec![], + }, + RejectionTestCase { + description: "a call with more arguments than any signature takes is rejected", + sql: "SELECT LENGTH(order_id, 2) AS x FROM orders; \ + SELECT hex_encode(status, 1, 2) FROM orders", + expected_rejections: vec!["too_many_arguments", "too_many_arguments"], + expected_codes: vec!["SQ0207", "SQ0207"], + }, + RejectionTestCase { + description: "calls within their limits, with named arguments, unbounded or unknown are not", + sql: "SELECT LENGTH(order_id), SUBSTR(status, 1, 2), COALESCE(a, b, c, d, e) FROM orders; \ + SELECT AI_EXTRACT(file => f, responseFormat => r, scores => TRUE) FROM orders; \ + SELECT NOSUCHFUNCTION(1, 2, 3, 4) FROM orders", + expected_rejections: vec![], + expected_codes: vec![], + }, + RejectionTestCase { + description: "a column a lone PIVOT aggregates or pivots on is rejected when read after it", + sql: "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('a', 'b')) AS p; \ + SELECT order_id FROM orders PIVOT (SUM(total) FOR status IN ('a')) AS p WHERE p.status = 'a'", + expected_rejections: vec!["pivot_consumed_column", "pivot_consumed_column"], + expected_codes: vec!["SQ0208", "SQ0208"], + }, + RejectionTestCase { + description: "pivoted values, aliases, joins and nested queries are not", + sql: "SELECT order_id, \"'a'\" FROM orders PIVOT (SUM(total) FOR status IN ('a')) AS p; \ + SELECT order_id AS total FROM orders PIVOT (SUM(total) FOR status IN ('a')) AS p ORDER BY total; \ + SELECT total FROM orders PIVOT (SUM(total) FOR status IN ('a')) AS p JOIN totals ON TRUE; \ + SELECT (SELECT total FROM totals) FROM orders PIVOT (SUM(total) FOR status IN ('a')) AS p", + expected_rejections: vec![], + expected_codes: vec![], + }, RejectionTestCase { description: "a star without ILIKE, or ILIKE without REPLACE or RENAME, is left to conversion", sql: "SELECT * REPLACE (price * 2 AS price) FROM orders; \ @@ -144,3 +193,19 @@ fn given_sql_snowflake_parses_then_rejects_when_refusing_then_each_rejection_is_ ); } } + +#[test] +fn given_the_syntax_catalogue_when_listing_function_arities_then_the_list_matches_its_grammars() { + let test_cases = [ArityListTestCase { + description: "snowflake_function_arities.txt holds each function every call grammar bounds", + expected_arities: SNOWFLAKE_FUNCTION_ARITIES, + }]; + for test_case in test_cases { + assert_eq!( + catalogue_function_arities(), + test_case.expected_arities, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-core/src/refusals/tests/test_types.rs b/crates/sqlquack-core/src/refusals/tests/test_types.rs index 722916fc9..636f4a6a2 100644 --- a/crates/sqlquack-core/src/refusals/tests/test_types.rs +++ b/crates/sqlquack-core/src/refusals/tests/test_types.rs @@ -42,3 +42,8 @@ pub(crate) struct RejectionTestCase { pub(crate) expected_rejections: Vec<&'static str>, pub(crate) expected_codes: Vec<&'static str>, } + +pub(crate) struct ArityListTestCase { + pub(crate) description: &'static str, + pub(crate) expected_arities: &'static str, +} diff --git a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs index 5b12f6a2d..1d40f671e 100644 --- a/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs +++ b/crates/sqlquack/src/analysis/tests/test_everyday_corpus.rs @@ -13,10 +13,7 @@ fn given_everyday_snowflake_queries_when_checked_and_converted_then_each_keeps_i // SQL Snowflake rejects (snowflake-receipts.json) that SQLQuack neither fails to parse nor // flags in its checks; each should become a parse or check error. expected_rejections_not_caught: vec![ - "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p", "SELECT order_id FROM orders WHERE ordered_at > 5", - "SELECT LENGTH(order_id, 2) AS x FROM orders", - "SELECT order_id FROM orders o, orders o", ], }]; diff --git a/crates/sqlquack/tests/cli/test_check.rs b/crates/sqlquack/tests/cli/test_check.rs index 9cc427be7..401e0142c 100644 --- a/crates/sqlquack/tests/cli/test_check.rs +++ b/crates/sqlquack/tests/cli/test_check.rs @@ -28,6 +28,22 @@ fn given_snowflake_sql_when_checked_then_problems_are_coded_and_the_exit_status_ expected_stdout_prefix: "", expected_stderr_mention: "error[SQ0204]", }, + CheckCommandTestCase { + description: "SQ0206 points at the alias written the second time", + arguments: &["check"], + sql: "SELECT order_id FROM orders o, orders o", + expected_exit_code: Some(1), + expected_stdout_prefix: "", + expected_stderr_mention: "1 | SELECT order_id FROM orders o, orders o\n | ^ ", + }, + CheckCommandTestCase { + description: "SQ0207 points at the call passing too many arguments", + arguments: &["check"], + sql: "SELECT LENGTH(order_id, 2) AS x FROM orders", + expected_exit_code: Some(1), + expected_stdout_prefix: "", + expected_stderr_mention: "1 | SELECT LENGTH(order_id, 2) AS x FROM orders\n | ^^^^^^^^^^^^^^^^^^^ the most arguments LENGTH takes is 1, and this call passes 2", + }, CheckCommandTestCase { description: "SQ0203 points at the QUALIFY clause, not the statement's first line", arguments: &["check"], diff --git a/website/src/content/docs/docs/diagnostics/reference.mdx b/website/src/content/docs/docs/diagnostics/reference.mdx index 59d93b677..f314360d6 100644 --- a/website/src/content/docs/docs/diagnostics/reference.mdx +++ b/website/src/content/docs/docs/diagnostics/reference.mdx @@ -33,6 +33,9 @@ renumbered. See [Reading diagnostics](/docs/diagnostics/overview/) for the outpu | [`SQ0203`](#sq0203) | `qualify_without_window` | QUALIFY needs a window function | | [`SQ0204`](#sq0204) | `star_filter_drops_named_column` | REPLACE or RENAME names a column the star's ILIKE pattern removed | | [`SQ0205`](#sq0205) | `unknown_function` | Snowflake has no function of this name | +| [`SQ0206`](#sq0206) | `duplicate_alias` | Two FROM items share one alias | +| [`SQ0207`](#sq0207) | `too_many_arguments` | The call passes more arguments than the function takes | +| [`SQ0208`](#sq0208) | `pivot_consumed_column` | The query reads a column its PIVOT consumed | ## Conversion @@ -321,3 +324,45 @@ The call names a function that is neither one of Snowflake's documented function ```sh echo "SELECT NOSUCHFUNCTION(order_id) AS x FROM orders" | sqlquack check --schema schema.json ``` + +### SQ0206 + +**Two FROM items share one alias** (`duplicate_alias`) + +Each table, subquery or call in one FROM clause and its joins needs a distinct alias, so Snowflake rejects a query that gives two of them the same one (Snowflake error 2027, duplicate alias). Unquoted aliases compare in upper case, as Snowflake stores them. Only aliases written with the items are compared. + +**Help:** give each FROM item its own alias + +**Example:** + +```sh +echo "SELECT order_id FROM orders o, orders o" | sqlquack check +``` + +### SQ0207 + +**The call passes more arguments than the function takes** (`too_many_arguments`) + +The call passes more arguments than any documented signature of the Snowflake function accepts, so Snowflake rejects the statement (Snowflake error 939, too many arguments). The limits come from the syntax catalogue's call grammars, and only functions whose every signature has a fixed number of plain arguments are checked. Calls with named arguments, and calls with too few arguments, are not checked. + +**Help:** remove the extra arguments; Snowflake's documentation lists each signature + +**Example:** + +```sh +echo "SELECT LENGTH(order_id, 2) AS x FROM orders" | sqlquack check +``` + +### SQ0208 + +**The query reads a column its PIVOT consumed** (`pivot_consumed_column`) + +PIVOT replaces the column it aggregates and the column after FOR with one column per listed value, so a query that still reads either by name fails in Snowflake (Snowflake error 904, invalid identifier). This is checked when the PIVOT is the only item in FROM, for names the SELECT list does not define as aliases. + +**Help:** read the pivoted value columns, or select the column in a subquery before pivoting + +**Example:** + +```sh +echo "SELECT order_id, total FROM orders PIVOT (SUM(total) FOR status IN ('shipped', 'pending')) AS p" | sqlquack check +``` From c9f4155f4eb07627e5740d5f7d237a5a2cd29ca0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:36:55 +0000 Subject: [PATCH 12/12] test(playground): read the repository's policy file in the engine test The examples test read public/playground/sqlquack.toml, which only the site build copies there, and CI tests the engine before it builds the site. It now reads sqlquack.toml at the repository root, as the other tests in the file do. --- website/tests/playground.test.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/tests/playground.test.mjs b/website/tests/playground.test.mjs index bc16e23f9..2d428bc99 100644 --- a/website/tests/playground.test.mjs +++ b/website/tests/playground.test.mjs @@ -102,7 +102,8 @@ test('every example stops at the stage it is meant to, under the default policy }; assert.deepEqual(Object.keys(EXAMPLES).sort(), Object.keys(expected).sort()); // The page starts on the repository's policy under a UTC session; its switch turns to strict. - const policy = readFileSync(new URL('../public/playground/sqlquack.toml', import.meta.url), 'utf8'); + // The repository's policy file, which the build copies to public/playground/ for the page. + const policy = readFileSync(new URL('../../sqlquack.toml', import.meta.url), 'utf8'); const modes = { repository: JSON.stringify({ policy, utc_session: true }), strict: undefined }; for (const [mode, settings] of Object.entries(modes)) { for (const [name, sql] of Object.entries(EXAMPLES)) {