From b666553562c522a581dc89ed030ebb14c3f469ea Mon Sep 17 00:00:00 2001 From: kvlonge <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:46:27 +0000 Subject: [PATCH 1/9] chore(analysis): validation only, Snowflake, SQLite and DuckDB arms on the native trait Not for merging. Puts the three arms that the SQL macro and SQL-test work runs on into the native route, and pins SQLQuack to a train head that carries the native parsers' type markers. Dropped when this branch is rebased onto a main that has the arms and a SQLQuack main pin. --- Cargo.lock | 40 ++- Cargo.toml | 8 +- crates/sqlbuild-analysis/Cargo.toml | 2 + .../analysis_route/_helpers/duckdb_dialect.rs | 51 ++++ .../src/analysis_route/_helpers/mod.rs | 4 + .../_helpers/snowflake_dialect.rs | 81 ++++++ .../analysis_route/_helpers/sqlite_dialect.rs | 50 ++++ .../analysis_route/_helpers/type_spelling.rs | 237 ++++++++++++++++++ .../src/analysis_route/constants.rs | 117 ++++++++- .../src/analysis_route/models.rs | 33 +++ .../analysis_route/tests/test_native_gap.rs | 6 +- 11 files changed, 613 insertions(+), 16 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/duckdb_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/snowflake_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/sqlite_dialect.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/_helpers/type_spelling.rs diff --git a/Cargo.lock b/Cargo.lock index 7d467327d..703d2c2ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4091,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4144,7 +4144,9 @@ dependencies = [ "sqlclean", "sqlquack-semantics", "sqlquack-syntax", + "sqlquack-syntax-duckdb", "sqlquack-syntax-postgres", + "sqlquack-syntax-sqlite", ] [[package]] @@ -4500,7 +4502,7 @@ dependencies = [ [[package]] name = "sqlquack-core" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ "serde", "serde_json", @@ -4510,19 +4512,21 @@ dependencies = [ [[package]] name = "sqlquack-semantics" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ + "regex", "serde", "serde_json", "sqlquack-core", "sqlquack-snowflake", "sqlquack-syntax", + "stacker", ] [[package]] name = "sqlquack-snowflake" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ "serde", "sqlquack-core", @@ -4531,7 +4535,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ "serde", "serde_json", @@ -4544,17 +4548,35 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "sqlquack-syntax-duckdb" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" +dependencies = [ + "sqlquack-syntax", + "stacker", +] + [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5#19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" +dependencies = [ + "sqlquack-syntax", + "stacker", +] + +[[package]] +name = "sqlquack-syntax-sqlite" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=87df54081f4471615780943855015e142a348689#87df54081f4471615780943855015e142a348689" dependencies = [ "sqlquack-syntax", "stacker", @@ -4651,7 +4673,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5087,7 +5109,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c4475e838..4bba0fa92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -177,10 +177,12 @@ sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } # Scopes, name resolution, types and lineage for the native analysis route. -sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "87df54081f4471615780943855015e142a348689" } # The lossless SQL tokenizer behind query fingerprints. -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } -sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "87df54081f4471615780943855015e142a348689" } +sqlquack-syntax-duckdb = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "87df54081f4471615780943855015e142a348689" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "87df54081f4471615780943855015e142a348689" } +sqlquack-syntax-sqlite = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "87df54081f4471615780943855015e142a348689" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 95c643ebd..32b5764d6 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -28,7 +28,9 @@ sqlbuild-sqltext.workspace = true sqlclean.workspace = true sqlquack-semantics.workspace = true sqlquack-syntax.workspace = true +sqlquack-syntax-duckdb.workspace = true sqlquack-syntax-postgres.workspace = true +sqlquack-syntax-sqlite.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/duckdb_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/duckdb_dialect.rs new file mode 100644 index 000000000..b03d844a4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/duckdb_dialect.rs @@ -0,0 +1,51 @@ +//! DuckDB's native module: SQLQuack's native DuckDB parser reads its SQL. Validation only. + +use sqlquack_semantics::dialects::constants::DUCKDB; +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_syntax_duckdb::{Parsed, StrictFailure, parse_strict}; + +use crate::analysis_route::_helpers::type_spelling::respelt; +use crate::analysis_route::constants::{ + DUCKDB_DIALECT, DUCKDB_TYPE_SPELLING, UNQUOTED_STUB_PREFIX, UNSCANNED_CONSTRUCT, +}; +use crate::analysis_route::models::{DuckDbDialect, ParserGap}; +use crate::analysis_route::types::NativeDialect; +use crate::type_system::main::normalize_text_type::normalize_text_type; +use crate::type_system::models::{NormalizedType, TypeDialect}; + +impl NativeDialect for DuckDbDialect { + fn name(&self) -> &'static str { + DUCKDB_DIALECT + } + + fn stub_prefix(&self) -> &'static str { + UNQUOTED_STUB_PREFIX + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql).map_err(|failure| match failure { + StrictFailure::Parse(error) => ParserGap { + construct: error.message.clone(), + message: error.message, + offset: error.position, + syntax: error.syntax, + }, + StrictFailure::Uncovered { position } => ParserGap { + construct: UNSCANNED_CONSTRUCT.to_owned(), + message: UNSCANNED_CONSTRUCT.to_owned(), + offset: position, + syntax: false, + }, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &DUCKDB + } + + /// The type as Polyglot normalised it, or `None` (unknown) where the text fallback can't match it. + fn normalize_type(&self, spelling: &str) -> Option { + respelt(spelling, &DUCKDB_TYPE_SPELLING) + .map(|respelt| normalize_text_type(&respelt, Some(TypeDialect::DuckDb))) + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs index 715122942..025894d47 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod clickhouse_types; +pub(crate) mod duckdb_dialect; pub(crate) mod native_analysis; +pub(crate) mod snowflake_dialect; +pub(crate) mod sqlite_dialect; +pub(crate) mod type_spelling; pub(crate) mod whole_gap; diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/snowflake_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/snowflake_dialect.rs new file mode 100644 index 000000000..1eff9c3fd --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/snowflake_dialect.rs @@ -0,0 +1,81 @@ +//! Snowflake's native module: SQLQuack's Snowflake parser reads its SQL. Validation only. + +use sqlquack_semantics::dialects::constants::SNOWFLAKE; +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_syntax::{ + Dialect, Error, Expression, ParseOptions, Parsed, StrictFailure, TokenizeFailure, parse_strict, +}; + +use crate::analysis_route::_helpers::type_spelling::respelt; +use crate::analysis_route::constants::{ + SNOWFLAKE_DIALECT, SNOWFLAKE_TYPE_SPELLING, UNQUOTED_STUB_PREFIX, UNSCANNED_CONSTRUCT, +}; +use crate::analysis_route::models::{ParserGap, SnowflakeDialect}; +use crate::analysis_route::types::NativeDialect; +use crate::type_system::main::normalize_text_type::normalize_text_type; +use crate::type_system::models::{NormalizedType, TypeDialect}; + +impl NativeDialect for SnowflakeDialect { + fn name(&self) -> &'static str { + SNOWFLAKE_DIALECT + } + + fn stub_prefix(&self) -> &'static str { + UNQUOTED_STUB_PREFIX + } + + fn parse(&self, sql: &str) -> Result { + shared_parsed(sql, Dialect::Snowflake) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &SNOWFLAKE + } + + /// The type as Polyglot normalised it, or `None` (unknown) where the text fallback can't match it. + fn normalize_type(&self, spelling: &str) -> Option { + respelt(spelling, &SNOWFLAKE_TYPE_SPELLING) + .map(|respelt| normalize_text_type(&respelt, Some(TypeDialect::Snowflake))) + } +} + +/// Every statement the shared parser reads from `sql` in `dialect`, or its first refusal; it +/// cannot tell invalid syntax from an unsupported construct, so no refusal is marked syntax. +pub(crate) fn shared_parsed(sql: &str, dialect: Dialect) -> Result { + let statements: Vec = parse_strict(sql, dialect, &ParseOptions::default()) + .map_err(|failure| match failure { + StrictFailure::Tokens( + TokenizeFailure::Uncovered { start, .. } | TokenizeFailure::Overlap { start, .. }, + ) => ParserGap { + construct: UNSCANNED_CONSTRUCT.to_owned(), + message: UNSCANNED_CONSTRUCT.to_owned(), + offset: start, + syntax: false, + }, + StrictFailure::Parse(error) + | StrictFailure::Tokens(TokenizeFailure::Tokenizer(error)) => { + let offset: usize = error.start().unwrap_or_default(); + let message: String = unplaced(error); + ParserGap { + construct: message.clone(), + message, + offset, + syntax: false, + } + } + })?; + Ok(Parsed { + statements, + gaps: Vec::new(), + }) +} + +/// The parser's message without its own line and column, which P026 reports from the offset. +fn unplaced(error: Error) -> String { + match error { + Error::Tokenize { message, .. } + | Error::Parse { message, .. } + | Error::Syntax { message, .. } => message, + other => other.to_string(), + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/sqlite_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/sqlite_dialect.rs new file mode 100644 index 000000000..942eadafa --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/sqlite_dialect.rs @@ -0,0 +1,50 @@ +//! SQLite's native module: SQLQuack's native SQLite parser reads its SQL. Validation only. + +use sqlquack_semantics::dialects::constants::SQLITE; +use sqlquack_semantics::dialects::types::DialectProfile; +use sqlquack_syntax_sqlite::{Parsed, StrictFailure, parse_strict}; + +use crate::analysis_route::_helpers::type_spelling::respelt; +use crate::analysis_route::constants::{ + SQLITE_DIALECT, SQLITE_TYPE_SPELLING, UNQUOTED_STUB_PREFIX, UNSCANNED_CONSTRUCT, +}; +use crate::analysis_route::models::{ParserGap, SqliteDialect}; +use crate::analysis_route::types::NativeDialect; +use crate::type_system::main::normalize_text_type::normalize_text_type; +use crate::type_system::models::NormalizedType; + +impl NativeDialect for SqliteDialect { + fn name(&self) -> &'static str { + SQLITE_DIALECT + } + + fn stub_prefix(&self) -> &'static str { + UNQUOTED_STUB_PREFIX + } + + fn parse(&self, sql: &str) -> Result { + parse_strict(sql).map_err(|failure| match failure { + StrictFailure::Parse(error) => ParserGap { + construct: error.message.clone(), + message: error.message, + offset: error.position, + syntax: error.syntax, + }, + StrictFailure::Uncovered { position } => ParserGap { + construct: UNSCANNED_CONSTRUCT.to_owned(), + message: UNSCANNED_CONSTRUCT.to_owned(), + offset: position, + syntax: false, + }, + }) + } + + fn profile(&self) -> &'static dyn DialectProfile { + &SQLITE + } + + /// The type as Polyglot normalised it, or `None` (unknown) where the text fallback can't match it. + fn normalize_type(&self, spelling: &str) -> Option { + respelt(spelling, &SQLITE_TYPE_SPELLING).map(|respelt| normalize_text_type(&respelt, None)) + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/type_spelling.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/type_spelling.rs new file mode 100644 index 000000000..1c94661f3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/type_spelling.rs @@ -0,0 +1,237 @@ +//! A type as written, respelt with the names SQLBuild's type fallback compares. + +use crate::analysis_route::constants::{FALLBACK_TIMESTAMP_WORD, MAX_TYPE_NAME_WORDS}; +use crate::analysis_route::models::TypeSpelling; + +/// `spelling` with each type name renamed by `rules`, or `None` when it is not one type. +pub(crate) fn respelt(spelling: &str, rules: &TypeSpelling) -> Option { + let mut reader = Reader { + source: spelling, + text: spelling.as_bytes(), + position: 0, + rules, + }; + let respelt: String = reader.data_type(true)?; + reader.blank(); + (reader.position == reader.text.len()).then_some(respelt) +} + +struct Reader<'a> { + source: &'a str, + text: &'a [u8], + position: usize, + rules: &'a TypeSpelling, +} + +impl Reader<'_> { + /// One type, renamed, with its parameters and array suffixes; an array's element is nested. + fn data_type(&mut self, top_level: bool) -> Option { + let start: usize = self.position; + let mut respelt: String = self.element(false)?; + if top_level && self.peek_after_blank() != Some(b'[') { + self.position = start; + respelt = self.element(true)?; + } + let parameterised: usize = respelt.len(); + while self.peek_after_blank() == Some(b'[') { + if !self.rules.arrays { + return None; + } + self.blank(); + let suffix: usize = self.position; + self.position += 1; + while self.peek().is_some_and(|byte| byte.is_ascii_digit()) { + self.position += 1; + } + if self.peek() != Some(b']') { + return None; + } + self.position += 1; + respelt.push_str(self.source.get(suffix..self.position)?); + } + let unsplit: bool = respelt.len() > parameterised || respelt.matches(')').count() > 1; + let misread: bool = unsplit && respelt.to_uppercase().contains(FALLBACK_TIMESTAMP_WORD); + (!(top_level && misread)).then_some(respelt) + } + + /// One type without array suffixes, renamed. + fn element(&mut self, top_level: bool) -> Option { + let (written, renamed) = self.name(top_level)?; + let mut respelt: String = renamed.clone(); + self.blank(); + if self.peek() != Some(b'(') && self.rules.bare_kept.contains(&written.as_str()) { + respelt.clone_from(&written); + } + if self.peek() == Some(b'(') { + if self.rules.verbatim.contains(&renamed.as_str()) { + respelt.push_str(&self.balanced()?); + } else { + let fielded: bool = self.rules.fielded.contains(&renamed.as_str()); + let arguments: Vec = self.arguments(fielded)?; + if top_level && self.rules.sizeless_whole.contains(&written.as_str()) { + return None; + } + if !self.rules.sizeless.contains(&written.as_str()) { + respelt = format!("{respelt}({})", arguments.join(", ")); + } + } + } + Some(respelt) + } + + /// The longest type name the rules rename, or one word as written, upper-cased. + fn name(&mut self, top_level: bool) -> Option<(String, String)> { + self.blank(); + let mut words: Vec<(String, usize)> = Vec::new(); + while words.len() < MAX_TYPE_NAME_WORDS { + self.blank_if(!words.is_empty()); + let word_start: usize = self.position; + self.position = self.word_end(); + if self.position == word_start { + break; + } + let word: String = self.source.get(word_start..self.position)?.to_uppercase(); + words.push((word, self.position)); + } + let (first, first_end) = words.first().cloned()?; + for length in (1..=words.len()).rev() { + let phrase: String = words[..length] + .iter() + .map(|(word, _)| word.as_str()) + .collect::>() + .join(" "); + let nested: &[(&str, &str)] = if top_level { + &[] + } else { + self.rules.nested_aliases + }; + if let Some((_, renamed)) = self + .rules + .aliases + .iter() + .chain(nested) + .find(|(from, _)| *from == phrase) + { + self.position = words[length - 1].1; + return Some((phrase, (*renamed).to_owned())); + } + } + self.position = first_end; + Some((first.clone(), first)) + } + + /// A parenthesised argument list: field and type pairs, or types and literals. + fn arguments(&mut self, fielded: bool) -> Option> { + self.position += 1; + let mut arguments: Vec = Vec::new(); + loop { + self.blank(); + let argument: String = if fielded { + let field: String = self.field()?; + format!("{field} {}", self.data_type(false)?) + } else if self + .peek() + .is_some_and(|byte| byte.is_ascii_digit() || byte == b'\'') + { + self.literal()? + } else { + self.data_type(false)? + }; + arguments.push(argument); + self.blank(); + match self.peek()? { + b',' => self.position += 1, + b')' => { + self.position += 1; + return Some(arguments); + } + _ => return None, + } + } + } + + /// A field name as written; a bare one outside ASCII is quoted, as Polyglot quotes it. + fn field(&mut self) -> Option { + let start: usize = self.position; + if self.peek() == Some(b'"') { + self.position += 1; + while self.peek()? != b'"' { + self.position += 1; + } + self.position += 1; + } else { + self.position = self.word_end(); + } + let field: &str = self.source.get(start..self.position)?; + match (field.is_empty(), field.starts_with('"') || field.is_ascii()) { + (true, _) => None, + (false, true) => Some(field.to_owned()), + (false, false) => Some(format!("\"{field}\"")), + } + } + + /// A number or a quoted string, as written. + fn literal(&mut self) -> Option { + let start: usize = self.position; + if self.peek() == Some(b'\'') { + self.position += 1; + while self.peek()? != b'\'' { + self.position += 1; + } + self.position += 1; + } else { + while self.peek().is_some_and(|byte| byte.is_ascii_digit()) { + self.position += 1; + } + } + self.source.get(start..self.position).map(str::to_owned) + } + + /// A parenthesised span copied as written, which the fallback reads as text. + fn balanced(&mut self) -> Option { + let start: usize = self.position; + let mut depth: usize = 0; + loop { + match self.peek()? { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + self.position += 1; + if depth == 0 { + break; + } + } + self.source.get(start..self.position).map(str::to_owned) + } + + /// Where the word at the position ends: letters and digits in any script, and underscores. + fn word_end(&self) -> usize { + let rest: &str = self.source.get(self.position..).unwrap_or_default(); + self.position + + rest + .find(|character: char| !(character.is_alphanumeric() || character == '_')) + .unwrap_or(rest.len()) + } + + fn peek(&self) -> Option { + self.text.get(self.position).copied() + } + + fn peek_after_blank(&self) -> Option { + self.text[self.position..] + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + } + + fn blank(&mut self) { + self.blank_if(true); + } + + fn blank_if(&mut self, skip: bool) { + while skip && self.peek().is_some_and(|byte| byte.is_ascii_whitespace()) { + self.position += 1; + } + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 9e622059a..ff7a4f436 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -1,9 +1,29 @@ //! The native dialects and the diagnostic a model the native parser cannot read takes. +use crate::analysis_route::models::{DuckDbDialect, SnowflakeDialect, SqliteDialect, TypeSpelling}; use crate::analysis_route::types::NativeDialect; /// Each dialect, by its adapter's analysis dialect name, whose SQL a native module analyses. -pub(crate) const NATIVE_DIALECTS: &[(&str, &dyn NativeDialect)] = &[]; +// Validation only: the arms' own pull requests bring these modules. +pub(crate) const NATIVE_DIALECTS: &[(&str, &dyn NativeDialect)] = &[ + (DUCKDB_DIALECT, &DuckDbDialect), + (SNOWFLAKE_DIALECT, &SnowflakeDialect), + (SQLITE_DIALECT, &SqliteDialect), +]; +/// DuckDB's (and MotherDuck's) analysis dialect name. +pub(crate) const DUCKDB_DIALECT: &str = "duckdb"; +/// Snowflake's analysis dialect name. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; +/// SQLite's analysis dialect name. +pub(crate) const SQLITE_DIALECT: &str = "sqlite"; +/// The stub prefix Snowflake and SQLite read as one unquoted name. +pub(crate) const UNQUOTED_STUB_PREFIX: &str = "__sqlbuild_"; +/// The word that makes SQLBuild's type fallback call a type it cannot split into name and flat parameters a timestamp. +pub(crate) const FALLBACK_TIMESTAMP_WORD: &str = "TIMESTAMP"; +/// The most words a type name spans, as in `TIMESTAMP WITH TIME ZONE`. +pub(crate) const MAX_TYPE_NAME_WORDS: usize = 4; +/// The construct of a byte the scanner reads as no token, blank or comment. +pub(crate) const UNSCANNED_CONSTRUCT: &str = "unscanned text"; /// The dialect a project with no analysis dialect is named by. pub(crate) const GENERIC_DIALECT: &str = "generic"; /// The gap of SQL that holds more than one statement. @@ -26,3 +46,98 @@ pub(crate) const ERROR_SEVERITY: &str = "error"; pub(crate) const MACRO_BODY_PREFIX: &str = "SELECT "; /// The gap of a macro body that is not one expression. pub(crate) const NOT_ONE_EXPRESSION: &str = "not one expression"; +/// DuckDB's type names as SQLBuild compares them. +pub(crate) const DUCKDB_TYPE_SPELLING: TypeSpelling = TypeSpelling { + aliases: &[ + ("BINARY", "BLOB"), + ("BLOB", "VARBINARY"), + ("BOOL", "BOOLEAN"), + ("BYTEA", "BLOB"), + ("CHAR", "TEXT"), + ("CHARACTER", "TEXT"), + ("CHARACTER VARYING", "TEXT"), + ("DOUBLE PRECISION", "DOUBLE"), + ("HUGEINT", "INT128"), + ("INT64", "BIGINT"), + ("INTEGER", "INT"), + ("NUMERIC", "DECIMAL"), + ("ROW", "STRUCT"), + ("STRING", "TEXT"), + ("TIME WITH TIME ZONE", "TIMETZ"), + ("TIMESTAMP WITH TIME ZONE", "TIMESTAMPTZ"), + ("UHUGEINT", "UINT128"), + ("UINT16", "USMALLINT"), + ("UINT32", "UINTEGER"), + ("UINT64", "UBIGINT"), + ("UINT8", "UTINYINT"), + ("VARBINARY", "BLOB"), + ("VARCHAR", "TEXT"), + ], + nested_aliases: &[("DATETIME", "TIMESTAMP"), ("NVARCHAR", "TEXT")], + sizeless: &["CHAR", "CHARACTER"], + sizeless_whole: &["CHAR", "CHARACTER", "TIMESTAMP"], + bare_kept: &["ROW"], + fielded: &["STRUCT", "UNION"], + verbatim: &["LIST"], + arrays: true, +}; +/// Snowflake's type names as SQLBuild compares them. +pub(crate) const SNOWFLAKE_TYPE_SPELLING: TypeSpelling = TypeSpelling { + aliases: &[ + ("BINARY VARYING", "VARBINARY"), + ("CHAR VARYING", "VARCHAR"), + ("CHARACTER", "CHAR"), + ("CHARACTER VARYING", "VARCHAR"), + ("DOUBLE PRECISION", "DOUBLE"), + ("INTEGER", "INT"), + ("NCHAR VARYING", "VARCHAR"), + ("NUMERIC", "DECIMAL"), + ("REAL", "FLOAT"), + ("TEXT", "VARCHAR"), + ("TIMESTAMP_LTZ", "TIMESTAMPLTZ"), + ("TIMESTAMP_NTZ", "TIMESTAMPNTZ"), + ("TIMESTAMP_TZ", "TIMESTAMPTZ"), + ], + nested_aliases: &[], + sizeless: &[], + sizeless_whole: &["TIMESTAMP", "TIMESTAMP_TZ", "TIMESTAMPTZ"], + bare_kept: &[], + fielded: &["OBJECT"], + verbatim: &[], + arrays: false, +}; +/// SQLite's type names as SQLBuild compares them. +pub(crate) const SQLITE_TYPE_SPELLING: TypeSpelling = TypeSpelling { + aliases: &[ + ("BOOL", "BOOLEAN"), + ("CHAR", "TEXT"), + ("CHARACTER", "TEXT"), + ("DOUBLE", "REAL"), + ("DOUBLE PRECISION", "REAL"), + ("INT", "INTEGER"), + ("NUMERIC", "DECIMAL"), + ("SMALLINT", "INTEGER"), + ("VARBINARY", "BLOB"), + ("VARCHAR", "TEXT"), + ], + nested_aliases: &[], + sizeless: &["CHAR", "CHARACTER"], + sizeless_whole: &[ + "BIGINT", + "CHAR", + "CHARACTER", + "DOUBLE", + "DOUBLE PRECISION", + "FLOAT", + "INT", + "INTEGER", + "MEDIUMINT", + "REAL", + "SMALLINT", + "TINYINT", + ], + bare_kept: &[], + fielded: &[], + verbatim: &[], + arrays: false, +}; diff --git a/crates/sqlbuild-analysis/src/analysis_route/models.rs b/crates/sqlbuild-analysis/src/analysis_route/models.rs index 7330e008d..07859ad3c 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/models.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/models.rs @@ -122,3 +122,36 @@ impl LineageSchema for StandInSchema { .then_some(self.columns.as_slice()) } } + +/// DuckDB and MotherDuck, whose type spellings the native route compares as Polyglot did. +#[derive(Clone, Copy, Debug)] +pub struct DuckDbDialect; + +/// Snowflake, whose SQL SQLQuack's Snowflake parser reads. +#[derive(Clone, Copy, Debug)] +pub struct SnowflakeDialect; + +/// SQLite, whose SQL SQLQuack's native SQLite parser reads. +#[derive(Clone, Copy, Debug)] +pub struct SqliteDialect; + +/// How a dialect's type names are respelt before SQLBuild's type fallback reads them. +#[derive(Clone, Copy, Debug)] +pub struct TypeSpelling { + /// Each name, upper-cased and single-spaced, and the name it is compared as. + pub aliases: &'static [(&'static str, &'static str)], + /// Further aliases that apply only inside another type, such as a list's element. + pub nested_aliases: &'static [(&'static str, &'static str)], + /// Names whose parameters are dropped when they are renamed. + pub sizeless: &'static [&'static str], + /// Names whose parameterised spelling, as a whole type, the fallback cannot spell as compared. + pub sizeless_whole: &'static [&'static str], + /// Names kept as written when they have no parameters. + pub bare_kept: &'static [&'static str], + /// Types whose parameters are named fields. + pub fielded: &'static [&'static str], + /// Types whose parameters are compared as written. + pub verbatim: &'static [&'static str], + /// Whether `[]` array suffixes are part of the dialect's type syntax. + pub arrays: bool, +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs index 5c395c6d1..e73bf1d2f 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_native_gap.rs @@ -18,17 +18,17 @@ fn given_any_dialect_when_routing_analysis_then_it_stays_on_the_legacy_path() { RouteTestCase { description: "snowflake", dialect: Some("snowflake"), - expected_legacy: true, + expected_legacy: false, }, RouteTestCase { description: "duckdb", dialect: Some("duckdb"), - expected_legacy: true, + expected_legacy: false, }, RouteTestCase { description: "sqlite", dialect: Some("sqlite"), - expected_legacy: true, + expected_legacy: false, }, RouteTestCase { description: "bigquery", From e45a922ff1be54ce999e7e04f282269f63c39b35 Mon Sep 17 00:00:00 2001 From: kvlonge <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 09:46:27 +0000 Subject: [PATCH 2/9] feat(sql-macros): parse SQL macros and SQL-test planning with the native parser SQL macro bodies and SQL-test planning parsed with Polyglot. Both now parse with the native parser that analysis_route gives the project's SQL analysis dialect, and a dialect with no native parser reports the feature as unavailable rather than falling back. - Macro bodies parse with the project's parser instead of trying eight Polyglot dialects. Declarations and USING slot reads are masked as names of the same length, so spans line up. A body that only names a type is read as a cast's type. - Macro probes type through NativeDialect::analyse over stand-in relations; returns, ANY-family, argument and slot checks read its output types, and type findings are SQLBuild's B210-B219. - SQL-test planning parses with sql_test_parser(dialect). The planning template reads relation marker calls as plain names, so Snowflake accepts them in FROM, and a list after IN is read as a list. - Polyglot's output-typing entry points, used only by SQL macros, and the token-based simple-template fast path are removed. --- .../compiler/_helpers/sql_tests/assembly.rs | 5 + .../_helpers/sql_tests/check_independence.rs | 49 ++- .../compiler/_helpers/sql_tests/constants.rs | 2 - .../_helpers/sql_tests/expected_columns.rs | 15 +- .../_helpers/sql_tests/helper_names.rs | 12 +- .../_helpers/sql_tests/helper_scope.rs | 6 +- .../compiler/_helpers/sql_tests/mock_reads.rs | 56 ++- .../src/compiler/_helpers/sql_tests/mod.rs | 2 - .../compiler/_helpers/sql_tests/planning.rs | 96 ++--- .../_helpers/sql_tests/relation_markers.rs | 103 ++++-- .../compiler/_helpers/sql_tests/rendering.rs | 15 +- .../_helpers/sql_tests/scenario_extraction.rs | 4 +- .../_helpers/sql_tests/simple_template.rs | 266 -------------- .../main/scenario_expected_columns.rs | 5 +- .../main/sql_scenario_classification.rs | 7 +- .../sqlbuild-analysis/src/compiler/models.rs | 2 + .../sqlbuild-analysis/src/compiler/tests.rs | 6 +- .../src/compiler/tests/helpers.rs | 150 +------- .../tests/relation_markers/helpers.rs | 145 ++++++++ .../compiler/tests/simple_template/helpers.rs | 114 ------ .../tests/test_sql_test_expected_columns.rs | 11 +- .../tests/test_sql_test_helper_references.rs | 1 + .../tests/test_sql_test_name_isolation.rs | 7 +- .../compiler/tests/test_sql_test_planning.rs | 16 +- .../tests/test_sql_test_relation_markers.rs | 65 +++- .../tests/test_sql_test_simple_template.rs | 340 ------------------ .../src/compiler/tests/test_types.rs | 19 - .../_helpers/annotated_types.rs | 47 --- .../_helpers/compatibility_types.rs | 35 -- .../main/annotated_output_types.rs | 55 --- .../src/query_analysis/main/mod.rs | 2 - .../src/query_analysis/main/output_types.rs | 38 -- .../src/query_analysis/mod.rs | 3 - .../src/query_analysis/types.rs | 6 - .../src/sql_macros/_helpers/parsing/body.rs | 154 +++++--- .../sql_macros/_helpers/parsing/captures.rs | 10 +- .../sql_macros/_helpers/parsing/parameters.rs | 97 ++++- .../_helpers/rendering/cte_rename.rs | 73 ++-- .../_helpers/rendering/expansion.rs | 41 ++- .../src/sql_macros/_helpers/typing/probes.rs | 132 +++++-- .../_helpers/typing/return_types.rs | 108 +++--- .../_helpers/typing/returns_check.rs | 141 +++----- .../sql_macros/_helpers/typing/typed_calls.rs | 31 +- .../src/sql_macros/constants.rs | 19 +- .../src/sql_macros/models.rs | 5 - .../src/sql_macros/tests/helpers.rs | 22 ++ .../src/sql_macros/tests/test_expansion.rs | 122 +++++-- .../src/sql_macros/tests/test_typed_calls.rs | 3 +- .../sqlbuild-analysis/src/sql_macros/types.rs | 7 + .../src/sql_test_attachment/tests/helpers.rs | 3 +- .../compile_driver/assembly_project.rs | 8 +- .../compile_driver/assembly_sql_tests.rs | 5 +- .../_helpers/input_loops/scenario_loop.rs | 11 +- .../_helpers/render/declaration_values.rs | 5 + .../_helpers/render/sql_macro_host.rs | 6 + .../_helpers/sql_test_glue/assembly.rs | 2 + .../_helpers/sql_test_glue/compiled_tests.rs | 4 +- src/sqlbuild/_native.pyi | 2 +- src/sqlbuild/compiler/compile/types.py | 1 + .../sql_macros/test_sql_macro_returns.py | 230 +++++++----- .../sql_macros/test_sql_macro_types.py | 54 ++- .../compiler/sql_test_glue/helpers.py | 2 +- .../compiler/planner/_helpers/helpers.py | 1 + .../sql_test_assembly/test_plan_test.py | 6 + 64 files changed, 1317 insertions(+), 1693 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs delete mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/relation_markers/helpers.rs delete mode 100644 crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs delete mode 100644 crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs delete mode 100644 crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs delete mode 100644 crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs delete mode 100644 crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs delete mode 100644 crates/sqlbuild-analysis/src/query_analysis/types.rs diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs index 53a4e5c28..4ea04d9ca 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/assembly.rs @@ -3,11 +3,13 @@ use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers::sql_tests::case_identity::{FingerprintFailure, case_fingerprint}; use crate::compiler::_helpers::sql_tests::macro_call_names::macro_call_names; use crate::compiler::_helpers::sql_tests::mock_reads::{ MockReadsRequest, mock_reading_helper_diagnostics, }; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::models::{ AssembledSqlTestFacts, SqlTestAssemblyBatch, SqlTestAssemblyFailure, SqlTestAssemblyModel, SqlTestAssemblyOutcome, SqlTestAssemblyPayload, SqlTestAssemblyTest, SqlTestHelperDiagnostic, @@ -37,6 +39,7 @@ pub(crate) fn assemble_batch(batch: &SqlTestAssemblyBatch) -> Vec { models: &'a [SqlTestAssemblyModel], macro_names: &'a [MacroNames], syntax: &'a LexicalSyntax, + parser: Option<&'static dyn NativeDialect>, } fn assemble( @@ -94,6 +98,7 @@ fn assemble( models, target_model_names: &target_model_names, syntax, + parser: project.parser, })?; target_model_names .iter() diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs index ae53d08a1..431bc1ea8 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs @@ -4,16 +4,15 @@ use std::collections::{BTreeSet, HashMap}; use sqlbuild_core::graph_walk::main::breadth_first_path::breadth_first_path; -use polyglot_sql::{Dialect, DialectType, Expression, ExpressionWalk}; use serde_json::Value; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; +use sqlquack_syntax::{Expression, ExpressionWalk, Parsed}; -use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options; +use crate::analysis_route::types::NativeDialect; use crate::compiler::constants::{ALIAS_FIELD, CTES_FIELD, NAME_FIELD}; const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; -const TABLE_KIND: &str = "table"; const IDENTIFIER_QUOTES: &[u8] = b"\"`"; /// The check CTEs of one file and the labels its messages name. @@ -23,6 +22,8 @@ pub(crate) struct IndependenceCheck<'a> { pub(crate) assertion_prefix: &'a str, pub(crate) file_label: &'a str, pub(crate) context_label: &'a str, + /// The project's native parser; without one, references come from the identifier scan. + pub(crate) parser: Option<&'a dyn NativeDialect>, } /// The first independence error of `check`, if any. @@ -55,7 +56,7 @@ pub(crate) fn independence_error( if !casefold(body).contains(&casefold(prohibited_prefix)) { continue; } - if let Some(nested) = defined_cte_names(body) + if let Some(nested) = defined_cte_names(check.parser, body) .into_iter() .find(|nested| nested.starts_with(prohibited_prefix)) { @@ -71,7 +72,7 @@ pub(crate) fn independence_error( } let mut dependencies: HashMap> = HashMap::new(); for (name, body) in check.ctes { - let references = match known_cte_references(body, &names_by_key, syntax) { + let references = match known_cte_references(check.parser, body, &names_by_key, syntax) { Ok(references) => references, Err(message) => return Some(message), }; @@ -113,21 +114,20 @@ fn casefold(text: &str) -> String { text.to_lowercase() } -/// `parse_one(sql, dialect="generic")` under SQLBuild's complexity guard, or `None` on error. -fn parse_one(sql: &str) -> Option { - let Ok(options) = guarded_parse_options() else { - return None; - }; - let Ok(mut statements) = Dialect::get(DialectType::Generic).parse_with_options(sql, &options) - else { - return None; - }; - (statements.len() == 1).then(|| statements.remove(0)) +/// The one statement the project's parser reads from `sql` whole, or `None`. +fn parse_one(parser: Option<&dyn NativeDialect>, sql: &str) -> Option { + match parser?.parse(sql) { + Ok(Parsed { + mut statements, + gaps, + }) if gaps.is_empty() && statements.len() == 1 => statements.pop(), + _ => None, + } } /// Every CTE alias defined anywhere in `sql`, in the order `to_dict()` lists them. -fn defined_cte_names(sql: &str) -> Vec { - let Some(parsed) = parse_one(sql) else { +fn defined_cte_names(parser: Option<&dyn NativeDialect>, sql: &str) -> Vec { + let Some(parsed) = parse_one(parser, sql) else { return Vec::new(); }; let Ok(value) = serde_json::to_value(&parsed) else { @@ -165,6 +165,7 @@ fn collect_cte_names(value: &Value, mut names: Vec) -> Vec { /// The keys of the file's CTEs that `sql` reads, in reading order. fn known_cte_references( + parser: Option<&dyn NativeDialect>, sql: &str, names_by_key: &HashMap, syntax: &LexicalSyntax, @@ -173,21 +174,19 @@ fn known_cte_references( if !names_by_key.keys().any(|key| folded.contains(key.as_str())) { return Ok(Vec::new()); } - let Some(parsed) = parse_one(sql) else { + let Some(parsed) = parse_one(parser, sql) else { return identifier_references(sql, names_by_key, syntax); }; let mut references: Vec = Vec::new(); - for table in parsed - .dfs() - .skip(1) - .filter(|node| node.variant_name() == TABLE_KIND) - { - references = push_known(references, casefold(table.get_name()), names_by_key); + for node in parsed.dfs().skip(1) { + if let Expression::Table(table) = node { + references = push_known(references, casefold(&table.name.name), names_by_key); + } } Ok(references) } -/// The identifier scan used when Polyglot cannot parse a body. +/// The identifier scan used when the project's parser cannot read a body. fn identifier_references( sql: &str, names_by_key: &HashMap, diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs deleted file mode 100644 index 5c74d8b07..000000000 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub(crate) const SIMPLE_TEMPLATE_DIALECTS: [&str; 2] = ["duckdb", "snowflake"]; -pub(crate) const MAX_TEMPLATE_DEPTH: usize = 32; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs index 3a021b67b..cac9bfccf 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/expected_columns.rs @@ -2,14 +2,17 @@ use std::collections::HashSet; -use polyglot_sql::expressions::{Identifier, Select}; -use polyglot_sql::{Dialect, Expression}; +use sqlquack_syntax::expressions::names::models::Identifier; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::types::NativeDialect; /// Return the authored output column identifiers of `sql`, or `None` when they are not explicit. -pub(crate) fn expected_columns(sql: &str, dialect: &Dialect) -> Option> { - let mut statements = match dialect.parse(sql) { - Ok(statements) => statements, - Err(_) => return None, +pub(crate) fn expected_columns(sql: &str, parser: &dyn NativeDialect) -> Option> { + let mut statements = match parser.parse(sql) { + Ok(Parsed { statements, gaps }) if gaps.is_empty() => statements, + _ => return None, }; if statements.len() != 1 { return None; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_names.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_names.rs index 77ed5bcf9..b31844252 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_names.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_names.rs @@ -2,7 +2,9 @@ use std::collections::{HashMap, HashSet}; -use polyglot_sql::{Dialect, Expression, ExpressionWalk}; +use sqlquack_syntax::{Expression, ExpressionWalk, Parsed}; + +use crate::analysis_route::types::NativeDialect; pub(crate) const HELPER_PREFIX: &str = "__helper__"; @@ -15,11 +17,11 @@ pub(crate) fn helper_cte_name(name: &str) -> String { pub(crate) fn rename_helper_references( sql: &str, helpers: &HashMap, - dialect: &Dialect, + parser: &dyn NativeDialect, ) -> Option { - let statements = match dialect.parse(sql) { - Ok(statements) => statements, - Err(_) => return None, + let statements = match parser.parse(sql) { + Ok(Parsed { statements, gaps }) if gaps.is_empty() => statements, + _ => return None, }; let [statement] = statements.as_slice() else { return None; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs index 6f042f9d3..0ecf99580 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -4,9 +4,9 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use sqlbuild_core::graph_walk::main::depth_first_postorder::depth_first_postorder; -use polyglot_sql::Dialect; use rustc_hash::FxHashSet; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::SliceDialect; use crate::compiler::_helpers::sql_tests::cte_sql::{with_leading_ctes, with_unique_ctes}; @@ -84,7 +84,7 @@ pub(crate) struct ScopeRequest<'a> { pub(crate) fixtures: &'a TestFixtures, pub(crate) patterns: &'a SqlTestPatterns, pub(crate) file_label: &'a str, - pub(crate) rename_dialect: Option<&'a Dialect>, + pub(crate) rename_dialect: Option<&'a dyn NativeDialect>, pub(crate) flat_dialect: bool, pub(crate) slice_dialect: SliceDialect, /// Helpers the compiler found the test reads; an unread helper calling a reference is left out. @@ -221,7 +221,7 @@ impl ScopeGraph { fn rewrite_readers( &self, readers: &[(String, &str)], - dialect: &Dialect, + dialect: &dyn NativeDialect, patterns: &SqlTestPatterns, ) -> Option> { let names: HashMap = self diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs index 97e790075..aace735a9 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs @@ -3,10 +3,6 @@ use std::collections::HashSet; use std::convert::Infallible; -use polyglot_sql::expressions::With; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, ParseOptions, -}; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::graph_walk::models::VisitOrder; use sqlbuild_core::graph_walk::types::NodeSet; @@ -15,7 +11,10 @@ use sqlbuild_core::text::main::is_word_char::is_word_char; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use sqlquack_syntax::expressions::queries::models::With; +use sqlquack_syntax::{Expression, ExpressionWalk, Parsed}; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers::sql_tests::planning::MOCK_CTE_PREFIXES; use crate::compiler::models::{ SqlTestAssemblyFailure, SqlTestAssemblyModel, SqlTestAssemblyModelPayload, @@ -32,8 +31,6 @@ const RELATION_KINDS: [&str; 5] = [ DBT_REF_KIND, TABLE_FUNCTION_KIND, ]; -/// The function call depth SQLBuild's Polyglot proxy allows `parse_one`. -const MAX_FUNCTION_CALL_DEPTH: usize = 512; type Deferrable = Result; @@ -52,6 +49,8 @@ pub(crate) struct MockReadsRequest<'a> { pub(crate) models: &'a [SqlTestAssemblyModel], pub(crate) target_model_names: &'a [String], pub(crate) syntax: &'a LexicalSyntax, + /// The project's native parser; without one, reads come from the token scan. + pub(crate) parser: Option<&'a dyn NativeDialect>, } /// A Python dict by string key: a repeated key keeps its first position and takes the new value. @@ -155,7 +154,7 @@ pub(crate) fn mock_reading_helper_diagnostics( .iter() .chain(&payload.assertion_ctes) .collect(); - let graph = cte_graph(&payload.authored_ctes, &readers)?; + let graph = cte_graph(request.parser, &payload.authored_ctes, &readers)?; let mut scanned: Vec<&str> = readers.iter().map(|cte| cte.sql_body.as_str()).collect(); for key in read_helper_keys(&graph) { if let Some(cte) = graph.cte(&key) { @@ -241,7 +240,11 @@ fn model_reference(reference: &SqlTestAssemblyReference) -> Reference { } /// Python's `sql_test_cte_graph`. -fn cte_graph<'a>(authored: &'a [SqlTestCte], readers: &[&SqlTestCte]) -> Deferrable> { +fn cte_graph<'a>( + parser: Option<&dyn NativeDialect>, + authored: &'a [SqlTestCte], + readers: &[&SqlTestCte], +) -> Deferrable> { let mut ctes: OrderedEntries<&'a SqlTestCte> = OrderedEntries::new(); for cte in authored { ctes.assign(fold(&cte.name), cte); @@ -250,7 +253,7 @@ fn cte_graph<'a>(authored: &'a [SqlTestCte], readers: &[&SqlTestCte]) -> Deferra let mut reads: OrderedEntries> = OrderedEntries::new(); for (key, cte) in &ctes.entries { let others: Vec = keys.iter().filter(|other| *other != key).cloned().collect(); - let cte_reads = match parsed_reads(&cte.sql_body, &others)? { + let cte_reads = match parsed_reads(parser, &cte.sql_body, &others) { Some(parsed) => parsed, None => token_reads(&cte.sql_body, &others), }; @@ -258,7 +261,7 @@ fn cte_graph<'a>(authored: &'a [SqlTestCte], readers: &[&SqlTestCte]) -> Deferra } let mut reader_reads: Vec = Vec::new(); for reader in readers { - let parsed = match parsed_reads(&reader.sql_body, &keys)? { + let parsed = match parsed_reads(parser, &reader.sql_body, &keys) { Some(parsed) => parsed, None => token_reads(&reader.sql_body, &keys), }; @@ -276,18 +279,24 @@ fn cte_graph<'a>(authored: &'a [SqlTestCte], readers: &[&SqlTestCte]) -> Deferra } /// Python's `_parsed_reads`: unqualified reads of `keys` no nested CTE shadows, or None unparsed. -fn parsed_reads(sql: &str, keys: &[String]) -> Deferrable>> { +fn parsed_reads( + parser: Option<&dyn NativeDialect>, + sql: &str, + keys: &[String], +) -> Option> { let folded = fold(sql); if !keys.iter().any(|key| folded.contains(key.as_str())) { - return Ok(Some(Vec::new())); + return Some(Vec::new()); } - let options = proxy_parse_options()?; - let Ok(mut statements) = Dialect::get(DialectType::Generic).parse_with_options(sql, &options) + let Ok(Parsed { + mut statements, + gaps, + }) = parser?.parse(sql) else { - return Ok(None); + return None; }; - if statements.len() != 1 { - return Ok(None); + if statements.len() != 1 || !gaps.is_empty() { + return None; } let parsed: Expression = statements.remove(0); let nested = defined_cte_keys(&parsed); @@ -304,18 +313,7 @@ fn parsed_reads(sql: &str, keys: &[String]) -> Deferrable>> { reads.push(key); } } - Ok(Some(reads)) -} - -/// The parse options SQLBuild's Polyglot proxy gives `parse_one`. -fn proxy_parse_options() -> Deferrable { - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - })) - .map_err(|error| SqlTestAssemblyFailure::Internal(error.to_string()))?; - Ok(ParseOptions { - complexity_guard: Some(guard), - }) + Some(reads) } /// Python's `_defined_cte_keys` over `to_dict()`. diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index 745b163bc..5c0b7d52b 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -2,7 +2,6 @@ pub mod assembly; pub mod authored_macro_calls; pub mod case_identity; pub(crate) mod check_independence; -pub(crate) mod constants; pub mod cte_rename; pub mod cte_slices; pub mod cte_sql; @@ -22,4 +21,3 @@ pub mod relation_markers; pub mod rendering; pub mod scenario_extraction; pub(crate) mod scenario_scope; -pub(crate) mod simple_template; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index cb2d4599a..c774b21cd 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -5,14 +5,15 @@ use std::convert::Infallible; use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use std::time::Instant; -use polyglot_sql::{Dialect, Expression}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use regex::Regex; #[cfg(test)] use serde::Serialize; use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::graph_walk::main::depth_first_postorder::depth_first_postorder; +use sqlquack_syntax::{Expression, Parsed}; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, strip_statement_terminators}; use crate::compiler::_helpers::sql_tests::cte_sql::{ @@ -34,9 +35,11 @@ use crate::compiler::_helpers::sql_tests::reader_references::{ use crate::compiler::_helpers::sql_tests::real_model_limit::{ RealModelChain, real_model_limit_message, }; -use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; +use crate::compiler::_helpers::sql_tests::relation_markers::{ + markers_as_names, named_marker_calls, +}; use crate::compiler::_helpers::sql_tests::rendering::{ - RenderRequest, render_comparison_sql, render_dialect, rendered_chain_steps, + RenderRequest, render_comparison_sql, rendered_chain_steps, sql_test_parser, }; use crate::compiler::models::{ SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainStep, SqlTestCte, SqlTestPlan, @@ -95,7 +98,7 @@ struct ProjectContext { requires_derived_table_aliases: bool, analysis_templates: Arc, patterns: SqlTestPatterns, - render_dialect: Arc, + parser: &'static dyn NativeDialect, rejects_nested_with: bool, max_real_models: Option, /// Each model query's reserved CTE name, since every test reading the model checks it. @@ -111,6 +114,8 @@ pub(crate) struct SqlTestPatterns { table_function: Regex, pub(crate) dbt_reference: Regex, pub(crate) test_reference: Regex, + /// Any relation marker call: `__name("x")`, or `__name("a", "b")` as `__dbt_ref` writes it. + pub(crate) relation_marker: Regex, pub(crate) lexical: LexicalSyntax, pub(crate) identifier: Regex, } @@ -127,6 +132,9 @@ impl SqlTestPatterns { r#"(?i)__dbt_ref\(\s*\"([^\"]+)\"\s*(?:,\s*\"([^\"]+)\"\s*)?\)"#, )?, test_reference: compile_pattern(r#"(?i)__(?:ref|source|seed|dbt_ref|table_fn)\("#)?, + relation_marker: compile_pattern( + r#"(?i)\b(__[a-z_]+)\(\s*"([^"]+)"\s*(?:,\s*"([^"]+)"\s*)?\)"#, + )?, lexical, identifier: compile_pattern(r"[A-Za-z_][A-Za-z0-9_$]*")?, }) @@ -319,7 +327,7 @@ struct AnalysisResolutionRequest<'a> { resolved_chain: &'a HashMap, functions: &'a HashMap, file_label: &'a str, - dialect_name: &'a str, + parser: &'a dyn NativeDialect, templates: &'a AnalysisTemplateCache, patterns: &'a SqlTestPatterns, } @@ -509,7 +517,18 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result pub(crate) fn plan_batch(request: SqlTestPlanBatch) -> Result { let render_sql = request.render_sql; let include_plan = request.include_plan; - let render_dialect = Arc::new(render_dialect(request.sql_analysis_dialect.as_deref())); + let Some(parser) = sql_test_parser(request.sql_analysis_dialect.as_deref()) else { + if request.tests.is_empty() { + return Ok(SqlTestPlanBatchOutcome { + plans: Vec::new(), + planning_ns: 0, + rendering_ns: 0, + }); + } + return Err(unavailable_sql_tests( + request.sql_analysis_dialect.as_deref(), + )); + }; let rejects_nested_with = SliceDialect::new(request.sql_analysis_dialect.as_deref()).rejects_nested_with(); let context = ProjectContext { @@ -539,7 +558,7 @@ pub(crate) fn plan_batch(request: SqlTestPlanBatch) -> Result { resolved_chain: self.analysis_resolved, functions: &context.functions, file_label: self.file_label, - dialect_name: &context.dialect, + parser: context.parser, templates: &context.analysis_templates, patterns: &context.patterns, })? @@ -1378,8 +1397,11 @@ fn real_model_inputs(request: &TopoSortRequest<'_>, name: &str) -> Vec { fn analyze_and_resolve_sql( request: AnalysisResolutionRequest<'_>, ) -> Result, String> { - let Some(template) = - analysis_template(request.query_sql, request.dialect_name, request.templates)? + let Some(template) = analysis_template( + request.query_sql, + (request.parser, request.patterns), + request.templates, + )? else { return Ok(None); }; @@ -1470,7 +1492,7 @@ fn analyze_and_resolve_sql( )?; let cte_body_sql = strip_statement_terminators( &functions_resolved, - SliceDialect::new(Some(request.dialect_name)), + SliceDialect::new(Some(request.parser.name())), ) .to_string(); let resolved_sql = with_leading_ctes(&generated_state.generated, &cte_body_sql); @@ -1482,43 +1504,25 @@ fn analyze_and_resolve_sql( })) } -#[cfg(test)] -thread_local! { - static TEMPLATE_AST_PARSES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -#[cfg(test)] -pub(crate) fn take_template_ast_parse_count() -> usize { - TEMPLATE_AST_PARSES.with(|count| count.replace(0)) -} - pub(crate) fn analysis_template( query_sql: &str, - dialect_name: &str, + reader: (&dyn NativeDialect, &SqlTestPatterns), templates: &AnalysisTemplateCache, ) -> Result, String> { cached_analysis_template(query_sql, templates, || { - Dialect::get_by_name(dialect_name).and_then(|dialect| { - if let Some(template) = - super::simple_template::simple_analysis_template(query_sql, dialect_name, &dialect) - { - return Some(template); - } - parsed_analysis_template(query_sql, &dialect) - }) + parsed_analysis_template(query_sql, reader) }) } -/// The full-parser path also serves as the differential oracle for the bounded recognizer. +/// The template the parser reads whole from `query_sql`, its relation markers read as names. pub(crate) fn parsed_analysis_template( query_sql: &str, - dialect: &Dialect, + (parser, patterns): (&dyn NativeDialect, &SqlTestPatterns), ) -> Option { - #[cfg(test)] - TEMPLATE_AST_PARSES.with(|count| count.set(count.get() + 1)); - let mut statements = match dialect.parse(query_sql) { - Ok(statements) => statements, - Err(_) => return None, + let (named_sql, named) = markers_as_names(query_sql, patterns); + let mut statements = match parser.parse(&named_sql) { + Ok(Parsed { statements, gaps }) if gaps.is_empty() => statements, + _ => return None, }; if statements.len() != 1 { return None; @@ -1526,10 +1530,20 @@ pub(crate) fn parsed_analysis_template( let expression = statements.remove(0); Some(AnalysisTemplate { existing_cte_names: top_level_cte_names(&expression), - marker_calls: relation_marker_calls(&expression).into(), + marker_calls: named_marker_calls(&expression, &named).into(), }) } +/// Why SQL tests cannot be planned for a project whose SQL analysis dialect has no native parser. +fn unavailable_sql_tests(dialect: Option<&str>) -> String { + match dialect { + Some(dialect) => format!( + "SQL tests are unavailable for SQL analysis dialect '{dialect}', which SQLBuild has no native parser for" + ), + None => "SQL tests are unavailable without a SQL analysis dialect".to_owned(), + } +} + pub(crate) fn cached_analysis_template( query_sql: &str, templates: &AnalysisTemplateCache, diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs index 8f8e991ae..daaaa50dc 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs @@ -1,63 +1,122 @@ //! Relation marker calls in a serialized pre-order walk of a parsed statement's relations. -use polyglot_sql::expressions::{From, Join}; -use polyglot_sql::{Expression, ExpressionWalk}; +use sqlquack_syntax::expressions::queries::models::{From, Join}; +use sqlquack_syntax::{Expression, ExpressionWalk}; -use crate::compiler::_helpers::sql_tests::planning::DBT_REF_FUNCTION; +use crate::compiler::_helpers::sql_tests::markers::ProtectedRanges; +use crate::compiler::_helpers::sql_tests::planning::{DBT_REF_FUNCTION, SqlTestPatterns}; pub(crate) type MarkerCall = (String, String); -/// Return marker calls in node order, where Polyglot children follow declared field order. -pub(crate) fn relation_marker_calls(expression: &Expression) -> Vec { - expression.dfs().fold(Vec::new(), node_marker_calls) +/// The name a relation marker call stands as while the planning template is parsed. +const MARKER_NAME_PREFIX: &str = "__sqlbuild_relation_"; + +/// Marker calls in node order, the relations `named` by `markers_as_names` read back as their calls. +pub(crate) fn named_marker_calls(expression: &Expression, named: &[MarkerCall]) -> Vec { + expression.dfs().fold(Vec::new(), |calls, node| { + node_marker_calls(calls, node, named) + }) +} + +/// `sql` with each relation marker call in code as a plain name, which Snowflake reads in FROM. +pub(crate) fn markers_as_names(sql: &str, patterns: &SqlTestPatterns) -> (String, Vec) { + let mut protected: ProtectedRanges<'_> = ProtectedRanges::new(&patterns.lexical, sql); + let mut named: String = String::with_capacity(sql.len()); + let mut calls: Vec = Vec::new(); + let mut cursor: usize = 0; + for captures in patterns.relation_marker.captures_iter(sql) { + let (Some(full), Some(function), Some(first)) = + (captures.get(0), captures.get(1), captures.get(2)) + else { + continue; + }; + if protected.contains(full.start()) { + continue; + } + let function_name: String = function.as_str().to_ascii_lowercase(); + let referenced_name: String = match (captures.get(3), function_name == DBT_REF_FUNCTION) { + (None, _) => first.as_str().to_owned(), + (Some(second), true) => format!("{}__{}", first.as_str(), second.as_str()), + (Some(_), false) => continue, + }; + named.push_str(&sql[cursor..full.start()]); + named.push_str(&format!("{MARKER_NAME_PREFIX}{}", calls.len())); + cursor = full.end(); + calls.push((function_name, referenced_name)); + } + named.push_str(&sql[cursor..]); + (named, calls) } /// FROM then JOIN marker calls of one node, plus the serialized `on` that traversal skips. -fn node_marker_calls(mut calls: Vec, node: &Expression) -> Vec { +fn node_marker_calls( + mut calls: Vec, + node: &Expression, + named: &[MarkerCall], +) -> Vec { match node { Expression::Select(select) => calls.extend( select .from .iter() - .flat_map(from_markers) - .chain(join_markers(&select.joins)), + .flat_map(|from| from_markers(from, named)) + .chain(join_markers(&select.joins, named)), ), - Expression::From(from) => calls.extend(from_markers(from)), - Expression::Delete(delete) => calls.extend(join_markers(&delete.joins)), + Expression::From(from) => calls.extend(from_markers(from, named)), + Expression::Delete(delete) => calls.extend(join_markers(&delete.joins, named)), Expression::JoinedTable(table) => { calls.extend( - marker(&table.left) + marker(&table.left, named) .into_iter() - .chain(join_markers(&table.joins)), + .chain(join_markers(&table.joins, named)), ); } Expression::Update(update) => calls.extend( update .from_clause .iter() - .flat_map(from_markers) - .chain(join_markers(&update.from_joins)), + .flat_map(|from| from_markers(from, named)) + .chain(join_markers(&update.from_joins, named)), ), Expression::DataDeletionProperty(property) => { - return property.on.dfs().fold(calls, node_marker_calls); + return property + .on + .dfs() + .fold(calls, |calls, node| node_marker_calls(calls, node, named)); } _ => {} } calls } -fn from_markers(from: &From) -> impl Iterator + '_ { - from.expressions.iter().filter_map(marker) +fn from_markers<'a>( + from: &'a From, + named: &'a [MarkerCall], +) -> impl Iterator + 'a { + from.expressions + .iter() + .filter_map(|expression| marker(expression, named)) } -fn join_markers(joins: &[Join]) -> impl Iterator + '_ { - joins.iter().filter_map(|join| marker(&join.this)) +fn join_markers<'a>( + joins: &'a [Join], + named: &'a [MarkerCall], +) -> impl Iterator + 'a { + joins.iter().filter_map(|join| marker(&join.this, named)) } -fn marker(expression: &Expression) -> Option { +fn marker(expression: &Expression, named: &[MarkerCall]) -> Option { match expression { - Expression::Alias(alias) => marker(&alias.this), + Expression::Alias(alias) => marker(&alias.this, named), Expression::Function(function) => function_marker(&function.name, &function.args), + Expression::Table(table) if table.schema.is_none() => table + .name + .name + .strip_prefix(MARKER_NAME_PREFIX) + .and_then(|index| match index.parse::() { + Ok(index) => named.get(index).cloned(), + Err(_) => None, + }), _ => None, } } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs index d1d2e2f72..eff8501ae 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs @@ -1,9 +1,10 @@ //! Deterministic comparison SQL rendering for planned SQL-native tests. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use std::collections::{HashMap, HashSet}; -use polyglot_sql::{Dialect, DialectType}; - use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_rename::{CteRename, rename_ctes}; use crate::compiler::_helpers::sql_tests::cte_slices::{ @@ -602,10 +603,12 @@ fn render_response(request: RenderRequest) -> Result { }) } -/// Resolve the SQL-analysis dialect used to read expected-row columns. -pub(crate) fn render_dialect(name: Option<&str>) -> Dialect { - name.and_then(Dialect::get_by_name) - .unwrap_or_else(|| Dialect::get(DialectType::Generic)) +/// The native parser that reads SQL tests in analysis dialect `name`; `None` leaves them unavailable. +pub(crate) fn sql_test_parser(name: Option<&str>) -> Option<&'static dyn NativeDialect> { + match analysis_route(name) { + AnalysisRoute::Native(parser) => Some(parser), + _ => None, + } } pub(crate) fn render_difference_sample_json(request_json: &str) -> Result { diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs index 35b06efd6..0fb49c2f7 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs @@ -4,6 +4,7 @@ use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ct use sqlbuild_scopes::relationship_names::models::{RelationshipSource, TopLevelCtes}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers::sql_tests::check_independence::{ IndependenceCheck, independence_error, }; @@ -24,7 +25,7 @@ const SCENARIO_CONTEXT: &str = "SQL scenario"; pub(crate) fn extract_scenario( sql: &str, file: &str, - syntax: &LexicalSyntax, + (syntax, parser): (&LexicalSyntax, Option<&dyn NativeDialect>), ) -> Result { let ctes = match scan_top_level_ctes(sql, file, RelationshipSource::Scenario, syntax) { TopLevelCtes::Failed(message) => return Err(message), @@ -37,6 +38,7 @@ pub(crate) fn extract_scenario( assertion_prefix: ASSERT_PREFIX, file_label: file, context_label: SCENARIO_CONTEXT, + parser, }, syntax, ) { diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs deleted file mode 100644 index 6709e3376..000000000 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs +++ /dev/null @@ -1,266 +0,0 @@ -//! Closed token productions for test-planning facts; unrecognized input uses the full parser. - -use std::collections::HashSet; - -use polyglot_sql::Dialect; -use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; -use polyglot_sql::tokens::{Token, TokenType}; - -use crate::compiler::_helpers::sql_tests::constants::{ - MAX_TEMPLATE_DEPTH, SIMPLE_TEMPLATE_DIALECTS, -}; -use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, DBT_REF_FUNCTION}; -use crate::compiler::_helpers::sql_tests::relation_markers::MarkerCall; - -/// Recognize bounded DuckDB/Snowflake productions; other syntax retains the existing parser. -pub(crate) fn simple_analysis_template( - sql: &str, - dialect_name: &str, - dialect: &Dialect, -) -> Option { - if !SIMPLE_TEMPLATE_DIALECTS.contains(&dialect_name) || sql.contains("/*") { - return None; - } - let tokens = match dialect.tokenize(sql) { - Ok(tokens) => tokens, - Err(_) => return None, - }; - let mut reader = TemplateReader { - tokens: &tokens, - position: 0, - }; - let facts = reader.query(0)?; - reader.take(TokenType::Semicolon); - if reader.position != tokens.len() { - return None; - } - Some(AnalysisTemplate { - existing_cte_names: facts.names, - marker_calls: facts.calls.into(), - }) -} - -struct QueryFacts { - names: HashSet, - calls: Vec, -} - -struct TemplateReader<'a> { - tokens: &'a [Token], - position: usize, -} - -impl<'a> TemplateReader<'a> { - fn take(&mut self, kind: TokenType) -> bool { - if self - .tokens - .get(self.position) - .is_some_and(|token| token.token_type == kind) - { - self.position += 1; - true - } else { - false - } - } - - fn require(&mut self, kind: TokenType) -> Option<()> { - self.take(kind).then_some(()) - } - - fn identifier(&mut self) -> Option<&'a str> { - let token = self.tokens.get(self.position)?; - if !matches!( - token.token_type, - TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Final - ) { - return None; - } - self.position += 1; - Some(&token.text) - } - - fn query(&mut self, depth: usize) -> Option { - if depth > MAX_TEMPLATE_DEPTH { - return None; - } - let mut names: HashSet = HashSet::new(); - let mut cte_calls: Vec = Vec::new(); - let has_with = self.take(TokenType::With); - if has_with { - loop { - names.insert(self.identifier()?.to_owned()); - self.require(TokenType::As)?; - self.require(TokenType::LParen)?; - cte_calls.extend(self.query(depth + 1)?.calls); - self.require(TokenType::RParen)?; - if !self.take(TokenType::Comma) { - break; - } - } - } - let mut calls = self.select(depth)?; - while self.take(TokenType::Union) { - if has_with { - return None; - } - self.require(TokenType::All)?; - if self.take(TokenType::By) { - let name = self.tokens.get(self.position)?; - if name.token_type != TokenType::Var || !name.text.eq_ignore_ascii_case("name") { - return None; - } - self.position += 1; - } - calls.extend(self.select(depth)?); - } - calls.extend(cte_calls); - Some(QueryFacts { names, calls }) - } - - fn select(&mut self, depth: usize) -> Option> { - self.require(TokenType::Select)?; - loop { - if !self.take(TokenType::Star) { - self.scalar(depth + 1)?; - if self.take(TokenType::As) { - self.identifier()?; - } - } - if !self.take(TokenType::Comma) { - break; - } - } - self.require(TokenType::From)?; - let mut calls: Vec = self.relation()?.into_iter().collect(); - loop { - let left = self.take(TokenType::Left); - if !self.take(TokenType::Join) { - if left { - return None; - } - break; - } - calls.extend(self.relation()?); - self.require(TokenType::On)?; - self.condition(depth + 1)?; - } - if self.take(TokenType::Where) { - self.condition(depth + 1)?; - } - Some(calls) - } - - fn relation(&mut self) -> Option> { - let name = self.identifier()?; - let call = if self.take(TokenType::LParen) { - let function = name.to_ascii_lowercase(); - if !matches!( - function.as_str(), - "__ref" | "__source" | "__seed" | "__dbt_ref" - ) { - return None; - } - let mut target = self.marker_argument()?.to_owned(); - if function == DBT_REF_FUNCTION && self.take(TokenType::Comma) { - target.push_str("__"); - target.push_str(self.marker_argument()?); - } - self.require(TokenType::RParen)?; - Some((function, target)) - } else { - None - }; - if self.take(TokenType::As) { - self.identifier()?; - } - Some(call) - } - - /// Require quoted Columns; unquoted arguments can invoke private dialect-specific function rules. - fn marker_argument(&mut self) -> Option<&'a str> { - let token = self.tokens.get(self.position)?; - if token.token_type != TokenType::QuotedIdentifier - || NO_PAREN_FUNCTION_NAMES.contains(token.text.to_ascii_uppercase().as_str()) - { - return None; - } - self.identifier() - } - - fn condition(&mut self, depth: usize) -> Option<()> { - self.scalar(depth)?; - self.require(TokenType::Eq)?; - self.scalar(depth) - } - - fn scalar(&mut self, depth: usize) -> Option<()> { - self.atom(depth)?; - while self.take(TokenType::Plus) - || self.take(TokenType::Dash) - || self.take(TokenType::Star) - || self.take(TokenType::Slash) - { - self.atom(depth)?; - } - Some(()) - } - - fn atom(&mut self, depth: usize) -> Option<()> { - if depth > MAX_TEMPLATE_DEPTH { - return None; - } - if self.take(TokenType::LParen) { - self.scalar(depth + 1)?; - return self.require(TokenType::RParen); - } - if self.take(TokenType::Cast) { - self.require(TokenType::LParen)?; - self.scalar(depth + 1)?; - self.require(TokenType::As)?; - let data_type = self.tokens.get(self.position)?; - if !matches!( - data_type.token_type, - TokenType::Var | TokenType::Int | TokenType::Double - ) || !["int", "integer", "double"] - .iter() - .any(|name| data_type.text.eq_ignore_ascii_case(name)) - { - return None; - } - self.position += 1; - return self.require(TokenType::RParen); - } - if let Some(token) = self.tokens.get(self.position) - && token.token_type == TokenType::Number - { - if !token.text.bytes().any(|byte| byte.is_ascii_digit()) - || !token - .text - .bytes() - .all(|byte| byte.is_ascii_digit() || byte == b'.') - || token.text.bytes().filter(|byte| *byte == b'.').count() > 1 - { - return None; - } - self.position += 1; - return Some(()); - } - let name = self.identifier()?; - if self.take(TokenType::LParen) { - if !name.eq_ignore_ascii_case("coalesce") { - return None; - } - self.scalar(depth + 1)?; - while self.take(TokenType::Comma) { - self.scalar(depth + 1)?; - } - self.require(TokenType::RParen) - } else { - if self.take(TokenType::Dot) { - self.identifier()?; - } - Some(()) - } - } -} diff --git a/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_columns.rs index a1cb1379c..7ff60c155 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_columns.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/scenario_expected_columns.rs @@ -3,7 +3,7 @@ use crate::compiler::_helpers::sql_tests::expected_columns::{ expected_columns, normalized_identifier, }; -use crate::compiler::_helpers::sql_tests::rendering::render_dialect; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::models::ScenarioExpectedColumns; /// The columns an expected CTE lists and how they differ from the known model output. @@ -13,7 +13,8 @@ pub fn scenario_expected_columns( dialect: Option<&str>, model_columns: Option<&[String]>, ) -> ScenarioExpectedColumns { - let listed: Option> = expected_columns(expected_sql, &render_dialect(dialect)); + let listed: Option> = + sql_test_parser(dialect).and_then(|parser| expected_columns(expected_sql, parser)); let (Some(listed_columns), Some(model_columns)) = (listed.as_deref(), model_columns) else { return ScenarioExpectedColumns { listed, diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs index 96857520d..034fa791a 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs @@ -2,14 +2,15 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::analysis_route::types::NativeDialect; use crate::compiler::_helpers; use crate::compiler::models::SqlScenarioCtes; -/// The scenario's classified CTEs, or its error. +/// The scenario's classified CTEs, or its error; `parser` is the project's native parser. pub fn extract_scenario( sql: &str, file_label: &str, - syntax: &LexicalSyntax, + (syntax, parser): (&LexicalSyntax, Option<&dyn NativeDialect>), ) -> Result { - _helpers::sql_tests::scenario_extraction::extract_scenario(sql, file_label, syntax) + _helpers::sql_tests::scenario_extraction::extract_scenario(sql, file_label, (syntax, parser)) } diff --git a/crates/sqlbuild-analysis/src/compiler/models.rs b/crates/sqlbuild-analysis/src/compiler/models.rs index 37775178d..c6c2e2bfe 100644 --- a/crates/sqlbuild-analysis/src/compiler/models.rs +++ b/crates/sqlbuild-analysis/src/compiler/models.rs @@ -191,6 +191,8 @@ pub struct SqlTestAssemblyBatch { pub models: Vec, pub tests: Vec, pub lexical_syntax: LexicalSyntax, + /// The project's SQL analysis dialect, whose native parser reads helper references. + pub sql_analysis_dialect: Option, } /// What assembly reads of one model: its name, macro dependencies and references. diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 7e1eda417..a2f830e31 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -2,12 +2,12 @@ mod helpers; #[path = "tests/planning/helpers.rs"] mod planning_helpers; +#[path = "tests/relation_markers/helpers.rs"] +mod relation_markers_helpers; #[path = "tests/test_scenario_expected_columns.rs"] mod scenario_expected_columns; #[path = "tests/test_scenario_helper_ctes.rs"] mod scenario_helper_ctes; -#[path = "tests/simple_template/helpers.rs"] -mod simple_template_helpers; #[path = "tests/single_read/helpers.rs"] mod single_read_helpers; #[path = "tests/test_sql_test_cte_rename.rs"] @@ -32,7 +32,5 @@ mod sql_test_planning; mod sql_test_relation_markers; #[path = "tests/test_sql_test_rendering.rs"] mod sql_test_rendering; -#[path = "tests/test_sql_test_simple_template.rs"] -mod sql_test_simple_template; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index dea0e94b0..2d2a6aa78 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -2,12 +2,11 @@ use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier, Mutex}; -use polyglot_sql::Dialect; use serde_json::{Value, json}; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, split_top_level_with}; -use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::main::sql_test_extraction::extract_batch_json; use crate::compiler::tests::test_types::ExtractionRuleTestCase; use crate::compiler::tests::test_types::{ @@ -673,8 +672,9 @@ pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { ) ); assert!( - polyglot_sql::Dialect::get(polyglot_sql::DialectType::DuckDB) - .parse(resolved_sql) + sql_test_parser(Some("duckdb")) + .expect("DuckDB parser") + .read_whole(resolved_sql) .is_ok(), "{resolved_sql}" ); @@ -1570,143 +1570,6 @@ pub(crate) fn generic_lexical_syntax() -> Value { }) } -type MarkerCalls = Option>; - -/// The original JSON-tree walk, kept as the oracle the direct relation walk must match. -fn json_relation_marker_calls(value: &Value) -> Vec<(String, String)> { - let from_markers = ["from", "from_clause"] - .into_iter() - .filter_map(|key| value.get(key)) - .filter_map(|from| from.get("expressions")) - .filter_map(Value::as_array) - .flatten() - .chain(value.get("left").filter(|_| value.get("joins").is_some())) - .filter_map(json_marker); - let join_markers = ["joins", "from_joins"] - .into_iter() - .filter_map(|key| value.get(key)) - .filter_map(Value::as_array) - .flatten() - .filter_map(|join| join.get("this")) - .filter_map(json_marker); - let children: Vec<&Value> = value - .as_object() - .map(|object| object.values().collect()) - .or_else(|| value.as_array().map(|values| values.iter().collect())) - .unwrap_or_default(); - from_markers - .chain(join_markers) - .chain(children.into_iter().flat_map(json_relation_marker_calls)) - .collect() -} - -fn json_marker(expression: &Value) -> Option<(String, String)> { - let object = expression.as_object()?; - object.get("alias").map_or_else( - || json_function_marker(object), - |alias| alias.get("this").and_then(json_marker), - ) -} - -fn json_function_marker(object: &serde_json::Map) -> Option<(String, String)> { - let function = object.get("function")?.as_object()?; - let function_name = function.get("name")?.as_str()?.to_ascii_lowercase(); - let names: Vec = function - .get("args")? - .as_array()? - .iter() - .map(|arg| { - arg.get("column")? - .get("name")? - .get("name")? - .as_str() - .map(str::to_string) - }) - .collect::>()?; - let referenced_name = (function_name == "__dbt_ref" && names.len() == 2) - .then(|| names.join("__")) - .or_else(|| (names.len() == 1).then(|| names[0].clone()))?; - Some((function_name, referenced_name)) -} - -/// Owned marker calls for comparison with collected calls. -pub(crate) fn owned_marker_calls(calls: &[(&str, &str)]) -> MarkerCalls { - Some( - calls - .iter() - .map(|(function, name)| ((*function).to_string(), (*name).to_string())) - .collect(), - ) -} - -/// Parse one statement and return the direct walk's and the JSON oracle's marker calls. -pub(crate) fn relation_markers_and_json_oracle( - dialect: &str, - sql: &str, -) -> (MarkerCalls, MarkerCalls) { - let mut statements = Dialect::get_by_name(dialect) - .expect("test dialect must exist") - .parse(sql) - .expect("test SQL must parse"); - assert_eq!(statements.len(), 1, "{sql}"); - let expression = statements.remove(0); - ( - Some(relation_marker_calls(&expression)), - serde_json::to_value(&expression) - .ok() - .map(|value| json_relation_marker_calls(&value)), - ) -} - -/// Return every generated relation shape whose collected markers differ from the JSON oracle. -pub(crate) fn relation_marker_oracle_mismatches(dialect: &str) -> Vec { - let relations: &[&str] = &[ - "__ref(\"orders\")", - "__source(\"raw_orders\") AS s", - "__dbt_ref(\"shop\", \"customers\") c", - "(SELECT * FROM __seed(\"regions\") JOIN __ref(\"stores\") ON TRUE) AS d", - "LATERAL (SELECT * FROM __ref(\"items\")) AS l", - "__udf(\"fn_orders\")", - "plain_table", - ]; - let wrappers = [ - "SELECT * FROM {a} JOIN {b} ON TRUE", - "WITH x AS (SELECT * FROM {a}) SELECT (SELECT 1 FROM {b}) AS v FROM x", - "SELECT * FROM {a} WHERE EXISTS (SELECT 1 FROM {b})", - "SELECT * FROM {a} UNION SELECT * FROM {b}", - "SELECT * FROM {a}, {b}", - "SELECT * FROM ({a} JOIN {b} ON TRUE)", - "SELECT * FROM {a} AS x LEFT JOIN LATERAL (SELECT * FROM {b}) AS y ON TRUE", - "SELECT * FROM (SELECT * FROM {a}) AS x CROSS JOIN {b} WHERE x.id IN (SELECT id FROM {a})", - "UPDATE orders SET amount = 1 FROM {a} JOIN {b} ON TRUE", - "DELETE FROM orders USING {a} JOIN {b} ON TRUE", - "INSERT INTO orders SELECT * FROM {a} JOIN {b} ON TRUE", - ]; - let parser = Dialect::get_by_name(dialect).expect("test dialect must exist"); - wrappers - .iter() - .flat_map(|wrapper| { - relations.iter().flat_map(move |first| { - relations - .iter() - .map(move |second| wrapper.replace("{a}", first).replace("{b}", second)) - }) - }) - .filter(|sql| { - parser - .parse(sql) - .unwrap_or_default() - .iter() - .any(|expression| { - Some(relation_marker_calls(expression)) - != serde_json::to_value(expression) - .ok() - .map(|value| json_relation_marker_calls(&value)) - }) - }) - .collect() -} - /// Dialects and their analysis modes; analysis reads BigQuery and Databricks markers as strings. const DIALECT_MODES: [(&str, &[bool]); 6] = [ ("duckdb", &[true, false]), @@ -1719,7 +1582,10 @@ const DIALECT_MODES: [(&str, &[bool]); 6] = [ /// Plan a fixture-read helper beside a model on every dialect and check names stay isolated. pub(crate) fn helper_names_are_isolated(shared: &SharedNameTestCase) -> bool { - for (dialect, modes) in DIALECT_MODES { + let planned = DIALECT_MODES + .into_iter() + .filter(|(dialect, _)| sql_test_parser(Some(dialect)).is_some()); + for (dialect, modes) in planned { for &sql_analysis_enabled in modes { let label = format!( "{} on {dialect}, analysis {sql_analysis_enabled}", diff --git a/crates/sqlbuild-analysis/src/compiler/tests/relation_markers/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/relation_markers/helpers.rs new file mode 100644 index 000000000..418e8c599 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/relation_markers/helpers.rs @@ -0,0 +1,145 @@ +//! The relation-marker walk checked against the original JSON-tree walk, its oracle. + +use serde_json::Value; + +use crate::compiler::_helpers::sql_tests::relation_markers::named_marker_calls; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; + +type MarkerCalls = Option>; + +/// The original JSON-tree walk, kept as the oracle the direct relation walk must match. +fn json_relation_marker_calls(value: &Value) -> Vec<(String, String)> { + let from_markers = ["from", "from_clause"] + .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(|from| from.get("expressions")) + .filter_map(Value::as_array) + .flatten() + .chain(value.get("left").filter(|_| value.get("joins").is_some())) + .filter_map(json_marker); + let join_markers = ["joins", "from_joins"] + .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(Value::as_array) + .flatten() + .filter_map(|join| join.get("this")) + .filter_map(json_marker); + let children: Vec<&Value> = value + .as_object() + .map(|object| object.values().collect()) + .or_else(|| value.as_array().map(|values| values.iter().collect())) + .unwrap_or_default(); + from_markers + .chain(join_markers) + .chain(children.into_iter().flat_map(json_relation_marker_calls)) + .collect() +} + +fn json_marker(expression: &Value) -> Option<(String, String)> { + let object = expression.as_object()?; + object.get("alias").map_or_else( + || json_function_marker(object), + |alias| alias.get("this").and_then(json_marker), + ) +} + +fn json_function_marker(object: &serde_json::Map) -> Option<(String, String)> { + let function = object.get("function")?.as_object()?; + let function_name = function.get("name")?.as_str()?.to_ascii_lowercase(); + let names: Vec = function + .get("args")? + .as_array()? + .iter() + .map(|arg| { + arg.get("column")? + .get("name")? + .get("name")? + .as_str() + .map(str::to_string) + }) + .collect::>()?; + let referenced_name = (function_name == "__dbt_ref" && names.len() == 2) + .then(|| names.join("__")) + .or_else(|| (names.len() == 1).then(|| names[0].clone()))?; + Some((function_name, referenced_name)) +} + +/// Owned marker calls for comparison with collected calls. +pub(crate) fn owned_marker_calls(calls: &[(&str, &str)]) -> MarkerCalls { + Some( + calls + .iter() + .map(|(function, name)| ((*function).to_string(), (*name).to_string())) + .collect(), + ) +} + +/// Parse one statement and return the direct walk's and the JSON oracle's marker calls. +pub(crate) fn relation_markers_and_json_oracle( + dialect: &str, + sql: &str, +) -> (MarkerCalls, MarkerCalls) { + let mut statements = sql_test_parser(Some(dialect)) + .expect("test dialect must exist") + .parse(sql) + .expect("test SQL must parse") + .statements; + assert_eq!(statements.len(), 1, "{sql}"); + let expression = statements.remove(0); + ( + Some(named_marker_calls(&expression, &[])), + serde_json::to_value(&expression) + .ok() + .map(|value| json_relation_marker_calls(&value)), + ) +} + +/// Return every generated relation shape whose collected markers differ from the JSON oracle. +pub(crate) fn relation_marker_oracle_mismatches(dialect: &str) -> Vec { + let relations: &[&str] = &[ + "__ref(\"orders\")", + "__source(\"raw_orders\") AS s", + "__dbt_ref(\"shop\", \"customers\") c", + "(SELECT * FROM __seed(\"regions\") JOIN __ref(\"stores\") ON TRUE) AS d", + "LATERAL (SELECT * FROM __ref(\"items\")) AS l", + "__udf(\"fn_orders\")", + "plain_table", + ]; + let wrappers = [ + "SELECT * FROM {a} JOIN {b} ON TRUE", + "WITH x AS (SELECT * FROM {a}) SELECT (SELECT 1 FROM {b}) AS v FROM x", + "SELECT * FROM {a} WHERE EXISTS (SELECT 1 FROM {b})", + "SELECT * FROM {a} UNION SELECT * FROM {b}", + "SELECT * FROM {a}, {b}", + "SELECT * FROM ({a} JOIN {b} ON TRUE)", + "SELECT * FROM {a} AS x LEFT JOIN LATERAL (SELECT * FROM {b}) AS y ON TRUE", + "SELECT * FROM (SELECT * FROM {a}) AS x CROSS JOIN {b} WHERE x.id IN (SELECT id FROM {a})", + "UPDATE orders SET amount = 1 FROM {a} JOIN {b} ON TRUE", + "DELETE FROM orders USING {a} JOIN {b} ON TRUE", + "INSERT INTO orders SELECT * FROM {a} JOIN {b} ON TRUE", + ]; + let parser = sql_test_parser(Some(dialect)).expect("test dialect must exist"); + wrappers + .iter() + .flat_map(|wrapper| { + relations.iter().flat_map(move |first| { + relations + .iter() + .map(move |second| wrapper.replace("{a}", first).replace("{b}", second)) + }) + }) + .filter(|sql| { + parser + .parse(sql) + .map(|parsed| parsed.statements) + .unwrap_or_default() + .iter() + .any(|expression| { + Some(named_marker_calls(expression, &[])) + != serde_json::to_value(expression) + .ok() + .map(|value| json_relation_marker_calls(&value)) + }) + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs deleted file mode 100644 index 8fb1f0df7..000000000 --- a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs +++ /dev/null @@ -1,114 +0,0 @@ -use polyglot_sql::Dialect; -use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; -use std::collections::HashMap; -use std::sync::Mutex; - -use crate::compiler::_helpers::sql_tests::planning::{ - AnalysisTemplate, analysis_template, parsed_analysis_template, -}; -use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; -use crate::compiler::tests::test_types::SimpleTemplateCorpusOutcome; - -pub(crate) fn no_paren_marker_mismatches(dialect_name: &str) -> Vec { - let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); - let cache = Mutex::new(HashMap::new()); - NO_PAREN_FUNCTION_NAMES - .iter() - .flat_map(|name| [name.to_string(), name.to_ascii_lowercase()]) - .flat_map(|name| [name.clone(), format!("\"{name}\"")]) - .flat_map(|argument| { - [ - format!("__ref({argument})"), - format!("__source({argument})"), - format!("__seed({argument})"), - format!("__dbt_ref({argument})"), - format!("__dbt_ref({argument}, \"orders\")"), - format!("__dbt_ref(\"orders\", {argument})"), - ] - }) - .filter_map(|relation| { - let sql = format!("SELECT id FROM {relation}"); - let fast = simple_analysis_template(&sql, dialect_name, &dialect); - let expected = parsed_analysis_template(&sql, &dialect); - let actual = analysis_template(&sql, dialect_name, &cache).expect("template cache"); - (fast.is_some() - || actual != expected - || expected - .as_ref() - .is_some_and(|facts| !facts.marker_calls.is_empty())) - .then(|| format!("{sql}: fast={fast:?}, actual={actual:?}, expected={expected:?}")) - }) - .collect() -} - -pub(crate) fn simple_template_corpus_outcome(dialect_name: &str) -> SimpleTemplateCorpusOutcome { - let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); - let mut queries: Vec = Vec::new(); - for projection in [ - "id", - "*", - "CAST(id AS INTEGER) AS id", - "COALESCE(id, 0) + id * 2 AS id", - "CAST((id + 1) / 2 AS DOUBLE) AS id", - "1.5 AS id", - "\"__ref(ignored)\" AS id", - "\"a\"\"b\" AS id", - ] { - for relation in [ - "__ref(\"orders\")", - "__source(\"orders\")", - "__seed(\"orders\")", - "__dbt_ref(\"shop\", \"orders\")", - "\"__ref\"(\"orders\")", - "__REF(orders)", - "\"__REF\"(\"orders\")", - "__ref(final)", - "__ref(\"a.b\")", - "__ref(\"a\"\"b\")", - ] { - let body = format!("SELECT {projection} FROM {relation}"); - for sql in [ - body.clone(), - format!("WITH recent AS ({body}) SELECT id FROM recent"), - format!("{body} UNION ALL BY NAME {body}"), - format!( - "WITH recent AS ({body}) SELECT id FROM __ref(\"customers\") AS c JOIN recent AS r ON c.id = r.id" - ), - ] { - let tokens = dialect.tokenize(&sql).expect("generated SQL tokenizes"); - for token in &tokens { - queries.push(format!( - "{} {}", - &sql[..token.span.start], - &sql[token.span.end..] - )); - for inserted in [".", ")", "(", "FROM", "'__ref(\"ignored\")'"] { - queries.push(format!( - "{} {inserted} {}", - &sql[..token.span.start], - &sql[token.span.start..] - )); - } - } - queries.push(sql); - } - } - } - let recognized: Vec<(String, AnalysisTemplate)> = queries - .into_iter() - .filter_map(|sql| { - simple_analysis_template(&sql, dialect_name, &dialect).map(|facts| (sql, facts)) - }) - .collect(); - SimpleTemplateCorpusOutcome { - recognized: recognized.len(), - mismatches: recognized - .into_iter() - .filter_map(|(sql, fast)| { - let parsed = parsed_analysis_template(&sql, &dialect); - (Some(&fast) != parsed.as_ref()) - .then(|| format!("{sql}: fast={fast:?}, parsed={parsed:?}")) - }) - .collect(), - } -} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs index 4e1752ed6..b95f97928 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_expected_columns.rs @@ -1,5 +1,5 @@ use crate::compiler::_helpers::sql_tests::expected_columns::expected_columns; -use crate::compiler::_helpers::sql_tests::rendering::render_dialect; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::tests::test_types::ExpectedColumnsTestCase; #[test] @@ -79,9 +79,12 @@ fn given_expected_cte_bodies_when_reading_columns_then_authored_names_or_none_ar }, ]; - for test_case in test_cases { - let dialect = render_dialect(Some(test_case.dialect)); - let actual: Option> = expected_columns(test_case.sql, &dialect); + for test_case in test_cases + .into_iter() + .filter(|test_case| sql_test_parser(Some(test_case.dialect)).is_some()) + { + let parser = sql_test_parser(Some(test_case.dialect)).expect("native parser"); + let actual: Option> = expected_columns(test_case.sql, parser); let actual_columns: Option> = actual .as_ref() .map(|columns| columns.iter().map(String::as_str).collect()); diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs index b6e1ffeaf..d8de1ba4c 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -589,6 +589,7 @@ fn given_compiled_sql_test_inputs_when_assembling_then_facts_match_python() { models: models(), tests, lexical_syntax: generic_syntax(), + sql_analysis_dialect: Some("duckdb".to_owned()), }; let macro_test = SqlTestAssemblyTest { block_name: None, diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_name_isolation.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_name_isolation.rs index 8c555fa0d..48fa7a353 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_name_isolation.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_name_isolation.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; -use polyglot_sql::Dialect; - use crate::compiler::_helpers::sql_tests::helper_names::rename_helper_references; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::tests::helpers::{helper_names_are_isolated, plan_shape_refusal}; use crate::compiler::tests::test_types::{ HelperRenameTestCase, NameRefusalTestCase, PlanShape, SharedNameTestCase, @@ -60,10 +59,10 @@ fn given_helper_references_when_renaming_then_only_relation_references_change() "expected_rows".to_string(), "__helper__expected_rows".to_string(), )]); - let dialect = Dialect::get_by_name("duckdb").expect("DuckDB dialect"); + let dialect = sql_test_parser(Some("duckdb")).expect("DuckDB parser"); for test_case in test_cases { assert_eq!( - rename_helper_references(test_case.sql, &helpers, &dialect).as_deref(), + rename_helper_references(test_case.sql, &helpers, dialect).as_deref(), test_case.expected_sql, "{}", test_case.description diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs index 43e3268d9..5f5403124 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs @@ -2,6 +2,7 @@ use serde_json::{Value, json}; use crate::compiler::_helpers::sql_tests::planning::plan_and_render_json; use crate::compiler::_helpers::sql_tests::planning::resolve_chains_json; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; use crate::compiler::main::sql_test_chains::resolve_sql_test_chains; use crate::compiler::main::sql_test_glue::plan_sql_test_batch; use crate::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; @@ -25,6 +26,7 @@ use crate::compiler::tests::test_types::{ /// A mocked model test, a test missing its source mock and a UDF test over two models. const GLUE_REQUEST: &str = r#"{ + "sqlAnalysisDialect": "duckdb", "lexicalSyntax": { "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] @@ -183,8 +185,8 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { let test_cases = [ GluePlanParityTestCase { - description: "generic SQL, planned without rendering", - dialect: None, + description: "DuckDB, planned without rendering", + dialect: Some("duckdb"), render_sql: false, expected_plan_count: 3, expected_missing_mock_severity: "error", @@ -217,7 +219,10 @@ fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry } value }; - for test_case in test_cases { + for test_case in test_cases + .into_iter() + .filter(|test_case| sql_test_parser(test_case.dialect).is_some()) + { let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); request["sqlAnalysisDialect"] = json!(test_case.dialect); request["renderSql"] = json!(test_case.render_sql); @@ -341,7 +346,10 @@ fn given_real_model_calling_udf_in_projection_when_planning_then_call_targets_th }, ]; - for test_case in test_cases { + for test_case in test_cases + .iter() + .filter(|test_case| sql_test_parser(Some(test_case.dialect)).is_some()) + { let sql = projected_udf_call_sql(test_case.dialect); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs index f22adc6a3..dd9f19d1a 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs @@ -1,4 +1,8 @@ -use crate::compiler::tests::helpers::{ +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiler::_helpers::sql_tests::planning::{SqlTestPatterns, parsed_analysis_template}; +use crate::compiler::_helpers::sql_tests::rendering::sql_test_parser; +use crate::compiler::tests::relation_markers_helpers::{ owned_marker_calls, relation_marker_oracle_mismatches, relation_markers_and_json_oracle, }; use crate::compiler::tests::test_types::{RelationMarkerOracleTestCase, RelationMarkersTestCase}; @@ -26,7 +30,7 @@ fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order }, RelationMarkersTestCase { description: "aliased joins keep from before joins", - dialect: "snowflake", + dialect: "duckdb", sql: "SELECT o.id FROM __ref(\"orders\") AS o JOIN __source(\"raw_customers\") AS c ON o.id = c.id LEFT JOIN __seed(\"regions\") r ON c.region = r.id", expected_calls: &[ ("__ref", "orders"), @@ -82,7 +86,10 @@ fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order expected_calls: &[("__ref", "orders"), ("__ref", "archived_orders")], }, ]; - for test_case in test_cases { + for test_case in test_cases + .iter() + .filter(|test_case| sql_test_parser(Some(test_case.dialect)).is_some()) + { let (collected, oracle) = relation_markers_and_json_oracle(test_case.dialect, test_case.sql); assert_eq!( @@ -134,7 +141,10 @@ fn given_generated_relation_shapes_when_collecting_markers_then_json_oracle_matc expected_mismatches: &[], }, ]; - for test_case in test_cases { + for test_case in test_cases + .iter() + .filter(|test_case| sql_test_parser(Some(test_case.dialect)).is_some()) + { assert_eq!( relation_marker_oracle_mismatches(test_case.dialect), test_case.expected_mismatches, @@ -143,3 +153,50 @@ fn given_generated_relation_shapes_when_collecting_markers_then_json_oracle_matc ); } } + +#[test] +fn given_marked_model_sql_when_reading_the_planning_template_then_every_parser_sees_the_markers() { + let sql: &str = "SELECT o.id, '__ref(\"quoted\")' AS q FROM __ref(\"orders\") AS o -- __ref(\"commented\")\nJOIN __source(\"raw_customers\") AS c ON o.id = c.id LEFT JOIN __seed(\"regions\") r ON c.region = r.id JOIN __dbt_ref(\"shop\", \"customers\") d ON TRUE WHERE o.id IN (SELECT id FROM __REF(\"Products\"))"; + let test_cases = [ + RelationMarkersTestCase { + description: "Snowflake, which takes a table function only inside TABLE( )", + dialect: "snowflake", + sql, + expected_calls: &[ + ("__ref", "orders"), + ("__source", "raw_customers"), + ("__seed", "regions"), + ("__dbt_ref", "shop__customers"), + ("__ref", "Products"), + ], + }, + RelationMarkersTestCase { + description: "DuckDB, the same calls in the same order", + dialect: "duckdb", + sql, + expected_calls: &[ + ("__ref", "orders"), + ("__source", "raw_customers"), + ("__seed", "regions"), + ("__dbt_ref", "shop__customers"), + ("__ref", "Products"), + ], + }, + ]; + let patterns: SqlTestPatterns = SqlTestPatterns::new(LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + }) + .expect("the SQL-test patterns compile"); + for test_case in test_cases { + let parser = sql_test_parser(Some(test_case.dialect)).expect(test_case.description); + let template = parsed_analysis_template(test_case.sql, (parser, &patterns)) + .expect(test_case.description); + assert_eq!( + Some(template.marker_calls.to_vec()), + owned_marker_calls(test_case.expected_calls), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs deleted file mode 100644 index a11546bfe..000000000 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs +++ /dev/null @@ -1,340 +0,0 @@ -use std::collections::HashMap; -use std::sync::Mutex; - -use polyglot_sql::Dialect; - -use crate::compiler::_helpers::sql_tests::planning::{ - analysis_template, parsed_analysis_template, take_template_ast_parse_count, -}; -use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; -use crate::compiler::tests::simple_template_helpers::{ - no_paren_marker_mismatches, simple_template_corpus_outcome, -}; -use crate::compiler::tests::test_types::{ - RelationMarkerOracleTestCase, SimpleTemplateCorpusTestCase, SimpleTemplateTestCase, -}; - -#[test] -fn given_registered_no_paren_arguments_when_planning_then_all_marker_positions_use_the_parser() { - let test_cases = [ - RelationMarkerOracleTestCase { - description: "all registry names in quoted and unquoted marker argument positions", - dialect: "duckdb", - expected_mismatches: &[], - }, - RelationMarkerOracleTestCase { - description: "all Snowflake registry names in quoted and unquoted marker argument positions", - dialect: "snowflake", - expected_mismatches: &[], - }, - ]; - for test_case in test_cases { - assert_eq!( - no_paren_marker_mismatches(test_case.dialect), - test_case.expected_mismatches, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_bounded_and_fallback_queries_when_planning_then_facts_and_ast_counts_match() { - let test_cases = [ - SimpleTemplateTestCase { - description: "Snowflake dense projections and union remove the template AST", - dialect: "snowflake", - sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), -projected AS (SELECT CAST(b.id AS INTEGER) AS id, -CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS amount FROM imported AS b), -combined AS (SELECT id, amount FROM projected UNION ALL BY NAME SELECT id, amount FROM projected WHERE id = 0), -final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) -SELECT id, amount FROM final"#, - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "Snowflake dense UDF projection retains the template AST", - dialect: "snowflake", - sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), -projected AS (SELECT CAST(b.id AS INTEGER) AS id, -CAST(__udf("amount")(b.amount) AS DOUBLE) AS amount FROM imported AS b) -SELECT id, amount FROM projected"#, - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "Snowflake QUALIFY remains on the full parser", - dialect: "snowflake", - sql: "SELECT id FROM __ref(\"orders\") QUALIFY ROW_NUMBER() OVER (ORDER BY id) = 1", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "quoted no-paren source argument retains parser Function classification", - dialect: "duckdb", - sql: "SELECT id FROM __source(\"now\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unquoted no-paren reference argument retains parser Function classification", - dialect: "duckdb", - sql: "SELECT id FROM __ref(NOW)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "first dbt argument with no-paren name retains parser classification", - dialect: "duckdb", - sql: "SELECT id FROM __dbt_ref(\"pi\", \"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "second dbt argument with no-paren name retains parser classification", - dialect: "duckdb", - sql: "SELECT id FROM __dbt_ref(\"orders\", \"getdate\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unquoted ordinary argument conservatively retains full parser", - dialect: "duckdb", - sql: "SELECT id FROM __seed(orders)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unquoted dialect-specific user argument retains full parser", - dialect: "duckdb", - sql: "SELECT id FROM __source(user)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "quoted user is an ordinary Column in the pinned parser", - dialect: "duckdb", - sql: "SELECT id FROM __source(\"user\")", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "dense CTE projections joins and union remove one cold template AST", - dialect: "duckdb", - sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), -lookup_0 AS (SELECT * FROM __source("customers")), -projected AS (SELECT CAST(b.id AS INTEGER) AS id, -CAST(b.amount + COALESCE(l0.amount, 0) AS DOUBLE) AS amount, -CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS quantity -FROM imported AS b LEFT JOIN lookup_0 AS l0 ON l0.id = b.id), -combined AS (SELECT id, amount, quantity FROM projected -UNION ALL BY NAME SELECT id, amount, quantity FROM projected WHERE id = 0), -final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) -SELECT id, amount FROM final"#, - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "nested CTEs follow outer relation then WITH traversal order", - dialect: "duckdb", - sql: "WITH recent AS (WITH nested AS (SELECT * FROM __source(\"orders\")) SELECT id FROM nested) SELECT id FROM recent JOIN __ref(\"customers\") AS c ON c.id = recent.id", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "root union has no Select CTE names", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\") UNION ALL SELECT id FROM __source(\"orders\")", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "quoted names escapes and line-comment marker lookalikes", - dialect: "duckdb", - sql: "WITH \"recent\" AS (SELECT * FROM __ref(\"or\"\"ders\")) SELECT id FROM \"recent\" -- __seed(\"ignored\")", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "two-part dbt references precede joined seeds", - dialect: "duckdb", - sql: "SELECT id FROM __dbt_ref(\"shop\", \"orders\") AS o JOIN __seed(\"regions\") AS r ON o.id = r.id", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "scalar nested subquery retains full parser", - dialect: "duckdb", - sql: "SELECT (SELECT id FROM __ref(\"orders\")) AS id", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "string marker lookalike retains full parser", - dialect: "duckdb", - sql: "SELECT '__ref(\"ignored\")' AS note FROM __ref(\"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "table function call retains full parser", - dialect: "duckdb", - sql: "SELECT id FROM __table_fn(\"orders\")(1)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "scalar function marker retains full parser", - dialect: "duckdb", - sql: "SELECT __udf(\"amount\")(id) AS id FROM __ref(\"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "WITH over UNION retains full parser AST attachment", - dialect: "duckdb", - sql: "WITH recent AS (SELECT id FROM orders) SELECT id FROM recent UNION ALL SELECT id FROM recent", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unterminated block comment retains full parser policy", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\") /* unclosed", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "nested block comments retain full parser policy", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\") /* outer /* __ref(\"ignored\") */ inner */", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "missing projection retains parser failure", - dialect: "duckdb", - sql: "SELECT FROM __ref(\"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "incomplete cast retains parser failure", - dialect: "duckdb", - sql: "SELECT CAST(id AS INTEGER FROM __ref(\"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "incomplete WHERE retains parser failure", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\") WHERE", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unterminated identifier retains parser failure", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "unterminated table identifier retains parser failure", - dialect: "duckdb", - sql: "SELECT id FROM \"orders", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "incomplete join retains parser failure", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\") LEFT", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "multiple statements retain textual fallback", - dialect: "duckdb", - sql: "SELECT id FROM __ref(\"orders\"); SELECT id FROM other", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "literal marker argument is not a Column", - dialect: "duckdb", - sql: "SELECT id FROM __ref('orders')", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "postgres dollar quoting retains its parser", - dialect: "postgres", - sql: "SELECT $$__ref(\"ignored\")$$ AS note FROM __ref(\"orders\")", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "snowflake ordinary quoted names use recognized facts", - dialect: "snowflake", - sql: "SELECT id FROM __ref(\"orders\")", - expected_ast_parses: 0, - }, - SimpleTemplateTestCase { - description: "bigquery backtick names retain its parser", - dialect: "bigquery", - sql: "SELECT id FROM __ref(`orders`)", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "tsql brackets retain its parser", - dialect: "tsql", - sql: "SELECT id FROM __ref([orders])", - expected_ast_parses: 1, - }, - SimpleTemplateTestCase { - description: "databricks quoting retains its parser", - dialect: "databricks", - sql: "SELECT id FROM __ref(`orders`)", - expected_ast_parses: 1, - }, - ]; - for test_case in test_cases { - let dialect = Dialect::get_by_name(test_case.dialect).expect("supported test dialect"); - take_template_ast_parse_count(); - let expected = parsed_analysis_template(test_case.sql, &dialect); - assert_eq!( - take_template_ast_parse_count(), - 1, - "{}", - test_case.description - ); - let fast = simple_analysis_template(test_case.sql, test_case.dialect, &dialect); - assert_eq!( - usize::from(fast.is_none()), - test_case.expected_ast_parses, - "{}", - test_case.description - ); - assert_eq!( - fast.as_ref(), - expected - .as_ref() - .filter(|_| test_case.expected_ast_parses == 0), - "{}", - test_case.description - ); - let cache = Mutex::new(HashMap::new()); - let actual = - analysis_template(test_case.sql, test_case.dialect, &cache).expect("template cache"); - assert_eq!(actual, expected, "{}", test_case.description); - assert_eq!( - take_template_ast_parse_count(), - test_case.expected_ast_parses, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_generated_and_mutated_queries_when_recognized_then_every_fact_matches_the_ast() { - let test_cases = [ - SimpleTemplateCorpusTestCase { - description: "combinations and token deletions preserve parser acceptance and facts", - dialect: "duckdb", - expected_min_recognized: 200, - expected_mismatches: &[], - }, - SimpleTemplateCorpusTestCase { - description: "Snowflake token productions preserve parser acceptance and facts", - dialect: "snowflake", - expected_min_recognized: 200, - expected_mismatches: &[], - }, - ]; - for test_case in test_cases { - let outcome = simple_template_corpus_outcome(test_case.dialect); - assert!( - outcome.recognized >= test_case.expected_min_recognized, - "{}: {} recognized", - test_case.description, - outcome.recognized - ); - assert_eq!( - outcome.mismatches, test_case.expected_mismatches, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index bf32b0365..7b561363c 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -186,25 +186,6 @@ pub(crate) struct SqlTestAssemblyTestCase { pub(crate) batch: crate::compiler::models::SqlTestAssemblyBatch, pub(crate) expected_outcomes: Vec, } -pub(crate) struct SimpleTemplateTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub sql: &'static str, - pub expected_ast_parses: usize, -} - -pub(crate) struct SimpleTemplateCorpusTestCase { - pub description: &'static str, - pub dialect: &'static str, - pub expected_min_recognized: usize, - pub expected_mismatches: &'static [&'static str], -} - -pub(crate) struct SimpleTemplateCorpusOutcome { - pub recognized: usize, - pub mismatches: Vec, -} - #[derive(Debug)] pub(crate) struct ScenarioExpectedColumnsTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs deleted file mode 100644 index 28a8d6b50..000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! Facts of a fully annotated query: which calls annotation could not type, and output types. - -use polyglot_sql::expressions::DataType; -use polyglot_sql::optimizer::annotate_types::function_result_is_known; -use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{DialectType, Expression}; - -/// Whether a call under `node`, outside a typed marker, has an unknown or only guessed type. -pub(crate) fn has_untyped_call(node: &Expression, dialect: DialectType) -> bool { - match node { - Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, - Expression::Function(function) - if !function_result_is_known(function, dialect) || is_unknown(node) => - { - true - } - Expression::AggregateFunction(_) | Expression::WindowFunction(_) if is_unknown(node) => { - true - } - _ => node - .children() - .into_iter() - .any(|child| has_untyped_call(child, dialect)), - } -} - -/// Whether annotation left `node` without a type, or with a wholly unknown one. -fn is_unknown(node: &Expression) -> bool { - node.inferred_type() - .is_none_or(|data_type| *data_type == DataType::Unknown) -} - -/// The annotated type of each projection of a `SELECT`, past its alias. -pub(crate) fn projection_types(expression: &Expression) -> Vec> { - let Expression::Select(select) = expression else { - return Vec::new(); - }; - let mut types: Vec> = Vec::with_capacity(select.expressions.len()); - for projection in &select.expressions { - let value: &Expression = match projection { - Expression::Alias(alias) => &alias.this, - other => other, - }; - types.push(value.inferred_type().cloned()); - } - types -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index ad8781052..a34c196d8 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -69,35 +69,11 @@ fn evaluate_outputs( functions, dialect, used_annotations: Cell::new(false), - untyped_source: Cell::new(false), }; let outputs = context.query(expression, &relations); (outputs, context.used_annotations.get()) } -/// Outputs only when every `FROM` and `JOIN` source of the query, CTEs included, was typed. -pub(super) fn infer_complete_outputs( - expression: &Expression, - schema: Option<&ValidationSchema>, - functions: &HashMap, - dialect: DialectType, -) -> Option { - let mut relations = Relations::new(); - if let Some(schema) = schema { - for table in &schema.tables { - relations.insert(table.name.to_lowercase(), Rc::new(table_outputs(table))); - } - } - let context = TypeContext { - functions, - dialect, - used_annotations: Cell::new(false), - untyped_source: Cell::new(false), - }; - let outputs = context.query(expression, &relations); - (!context.untyped_source.get()).then_some(outputs) -} - /// A schema table's columns with their declared types. fn table_outputs(table: &polyglot_sql::SchemaTable) -> Outputs { table @@ -111,8 +87,6 @@ struct TypeContext<'a> { functions: &'a HashMap, dialect: DialectType, used_annotations: Cell, - /// Whether a `FROM` or `JOIN` source could not be typed, so stars and columns may be missing. - untyped_source: Cell, } impl TypeContext<'_> { @@ -188,15 +162,6 @@ impl TypeContext<'_> { .flat_map(|from| &from.expressions) .chain(select.joins.iter().map(|join| &join.this)); for source in tables { - let typed: bool = match unwrapped(source) { - Expression::Values(_) => true, - Expression::Subquery(subquery) => subquery.alias.is_some(), - Expression::Table(table) => relations.contains_key(&table.name.name.to_lowercase()), - _ => false, - }; - if !typed { - self.untyped_source.set(true); - } if let Expression::Values(values) = unwrapped(source) { aliases.items.push(( values diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs deleted file mode 100644 index 45e65c7c9..000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::collections::HashMap; - -use polyglot_sql::expressions::DataType; -use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; - -use crate::query_analysis::annotated_types::{has_untyped_call, projection_types}; -use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_outputs}; -use crate::query_analysis::types::AnnotatedOutput; - -/// Each output of the one query in `sql`, typed by full annotation; `None` if a call is untyped. -#[must_use] -pub fn annotated_output_types( - sql: &str, - dialect: &str, - schema: &ValidationSchema, -) -> Option> { - let dialect: DialectType = match dialect.parse() { - Ok(dialect) => dialect, - Err(_) => return None, - }; - let mut statements: Vec = - match polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()) { - Ok(statements) => statements, - Err(_) => return None, - }; - if statements.len() != 1 { - return None; - } - let mut expression: Expression = statements.pop()?; - let mapping = polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( - schema, dialect, - ); - polyglot_sql::optimizer::annotate_types::annotate_types( - &mut expression, - Some(&mapping as &dyn polyglot_sql::schema::Schema), - Some(dialect), - ); - if has_untyped_call(&expression, dialect) { - return None; - } - let annotated: Vec> = projection_types(&expression); - let mut outputs: Vec = Vec::new(); - for (index, (name, data_type)) in - infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) - .into_iter() - .enumerate() - { - outputs.push(( - name, - data_type.filter(|value| value != NULL_TYPE), - annotated.get(index).cloned().flatten(), - )); - } - Some(outputs) -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs index 5d65d9ca3..926992194 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -1,3 +1 @@ -pub mod annotated_output_types; -pub mod output_types; pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs deleted file mode 100644 index c7d83d7d8..000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::collections::HashMap; - -use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; - -use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_complete_outputs}; - -/// The output columns and inferred types of the one query in `sql`; `None` if a source is untyped. -#[must_use] -pub fn output_types( - sql: &str, - dialect: &str, - schema: &ValidationSchema, -) -> Option)>> { - let dialect: DialectType = match dialect.parse() { - Ok(dialect) => dialect, - Err(_) => return None, - }; - let mut statements: Vec = - match polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()) { - Ok(statements) => statements, - Err(_) => return None, - }; - if statements.len() != 1 { - return None; - } - let expression: Expression = statements.pop()?; - let mut outputs: Vec<(String, Option)> = Vec::new(); - let inferred = infer_complete_outputs(&expression, Some(schema), &HashMap::new(), dialect)?; - for (name, data_type) in inferred { - outputs.push((name, data_type.filter(|value| !is_null_type(value)))); - } - Some(outputs) -} - -/// Whether `value` is the placeholder inference uses for a `NULL` literal's type. -fn is_null_type(value: &str) -> bool { - value == NULL_TYPE -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/mod.rs index 847eabad9..94f93ea62 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/mod.rs @@ -1,5 +1,3 @@ -#[path = "_helpers/annotated_types.rs"] -mod annotated_types; #[path = "_helpers/borrowed_facts.rs"] mod borrowed_facts; #[path = "_helpers/compatibility_types.rs"] @@ -7,7 +5,6 @@ mod compatibility_types; #[path = "_helpers/engine.rs"] mod engine; pub mod main; -pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/query_analysis/types.rs b/crates/sqlbuild-analysis/src/query_analysis/types.rs deleted file mode 100644 index ff1b963f8..000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Shapes of query analysis results. - -use polyglot_sql::expressions::DataType; - -/// One output of a fully annotated query: its name, inferred type, and annotated type. -pub type AnnotatedOutput = (String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs index fa92f6a59..3aba3eec4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs @@ -1,13 +1,20 @@ //! A SQL macro body parsed as SQL, its parameter references located and replaced in place. -use polyglot_sql::expressions::{Identifier, Select}; -use polyglot_sql::{DialectType, Expression, ExpressionWalk}; +use sqlquack_syntax::expressions::names::models::{Column, Identifier}; +use sqlquack_syntax::expressions::operators::models::DotAccess; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::{Expression, ExpressionWalk, Parsed}; + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use crate::sql_macros::_helpers::parsing::captures::{output_selects, slot_captures}; use crate::sql_macros::_helpers::parsing::header::unquoted; use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; use crate::sql_macros::_helpers::parsing::parameters::{ - ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, + ParameterReference, alias_follows, declarations_masked, in_lists_parenthesised, + parameter_references, placeholder, placeholder_parameter, using_reads_joined, with_placeholders, }; use crate::sql_macros::_helpers::parsing::slots::{SlotRead, check_group_positions, slot_reads}; @@ -22,18 +29,8 @@ use crate::sql_macros::models::{ SqlMacroDefinition, SqlMacroForm, SqlMacroParameter, TextSplice, TypedCall, }; -/// Dialects tried in turn to parse a body; the first that parses it locates its references. -const PARSE_DIALECTS: [DialectType; 8] = [ - DialectType::Generic, - DialectType::DuckDB, - DialectType::Snowflake, - DialectType::BigQuery, - DialectType::PostgreSQL, - DialectType::Databricks, - DialectType::ClickHouse, - DialectType::TSQL, -]; const EXPRESSION_PREFIX: &str = "SELECT "; +const TYPE_PREFIX: &str = "SELECT CAST(NULL AS "; const CTE_SUFFIX: &str = "\nSELECT 1"; /// The body parsed as SQL. @@ -41,30 +38,57 @@ pub(crate) struct ParsedBody { pub(crate) select: Box