diff --git a/Cargo.lock b/Cargo.lock index ba459c5fd0..39f628263f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4500,7 +4500,7 @@ dependencies = [ [[package]] name = "sqlquack-core" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "serde", "serde_json", @@ -4510,7 +4510,7 @@ dependencies = [ [[package]] name = "sqlquack-semantics" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "serde", "serde_json", @@ -4522,7 +4522,7 @@ dependencies = [ [[package]] name = "sqlquack-snowflake" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "serde", "sqlquack-core", @@ -4531,7 +4531,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "serde", "serde_json", @@ -4544,7 +4544,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-clickhouse" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "sqlquack-syntax", "stacker", @@ -4553,7 +4553,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=f48808e9b3ea7c3f928a758459a2b55fff7301a3#f48808e9b3ea7c3f928a758459a2b55fff7301a3" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de#7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index e6362fa24d..5897c1eb16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,9 +86,9 @@ proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } rpassword = "7.5.4" rcgen = "0.14" -sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f48808e9b3ea7c3f928a758459a2b55fff7301a3" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f48808e9b3ea7c3f928a758459a2b55fff7301a3" } -sqlquack-syntax-clickhouse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "f48808e9b3ea7c3f928a758459a2b55fff7301a3" } +sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" } +sqlquack-syntax-clickhouse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "7e5e1b1d485b74043a5cbf0f2cdb3eb73316f6de" } polyglot-sql = { git = "https://github.com/sqlstack-io/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs index 1aad3d0e26..3b4ca630eb 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs @@ -7,9 +7,7 @@ use sqlquack_syntax::Parsed; use sqlquack_syntax_clickhouse::parse_strict; use crate::analysis_route::_helpers::clickhouse_spelling::clickhouse_spelling; -use crate::analysis_route::constants::{ - CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX, STATEMENTS_CONSTRUCT, -}; +use crate::analysis_route::constants::{CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX}; use crate::analysis_route::main::clickhouse_type_match::clickhouse_type_match; use crate::analysis_route::models::{ClickHouseDialect, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; @@ -20,19 +18,11 @@ impl NativeDialect for ClickHouseDialect { CLICKHOUSE_DIALECT } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { - let parsed: Parsed = self.parse(sql)?; - match (parsed.gaps.first(), parsed.statements.len()) { - (Some(gap), _) => Err(ParserGap { - construct: gap.construct.to_owned(), - offset: gap.start, - }), - (None, 1) => Ok(()), - (None, _) => Err(ParserGap { - construct: STATEMENTS_CONSTRUCT.to_owned(), - offset: sql.len(), - }), - } + fn parse(&self, sql: &str) -> Result { + parse_strict(sql).map_err(|error| ParserGap { + construct: error.message, + offset: error.position, + }) } fn stub_prefix(&self) -> &'static str { @@ -41,14 +31,6 @@ impl NativeDialect for ClickHouseDialect { } impl ClickHouseDialect { - /// Every statement of `sql` with the parser's named gaps, or the first error. - pub fn parse(&self, sql: &str) -> Result { - parse_strict(sql).map_err(|error| ParserGap { - construct: error.message, - offset: error.position, - }) - } - /// The ClickHouse profile: names compared with case, types as ClickHouse 26.1 reports them. #[must_use] pub fn profile(&self) -> &'static dyn DialectProfile { diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index b6e3e52cb5..687a24498b 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -19,6 +19,8 @@ pub(crate) const CLICKHOUSE_DIALECT: &str = "clickhouse"; pub(crate) const CLICKHOUSE_STUB_PREFIX: &str = "__sqlbuild_"; /// The construct of SQL holding more than one statement. pub(crate) const STATEMENTS_CONSTRUCT: &str = "more than one statement"; +/// The construct of a statement a parser kept as unparsed text. +pub(crate) const UNPARSED_STATEMENT_CONSTRUCT: &str = "unparsed statement"; /// Polyglot's ClickHouse respelling of each SQL type alias (`{}` keeps the arguments) and family. pub(crate) const CLICKHOUSE_TYPE_ALIASES: &[(&str, &str, Option)] = &[ ("BOOLEAN", "Bool", None), diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index f65158f54e..e502487ade 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -1,8 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_clickhouse_dialect.rs"] mod test_clickhouse_dialect; #[path = "tests/test_clickhouse_type_match.rs"] mod test_clickhouse_type_match; #[path = "tests/test_native_gap.rs"] mod test_native_gap; +#[path = "tests/test_planner_routes.rs"] +mod test_planner_routes; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs new file mode 100644 index 0000000000..20db62abd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/helpers.rs @@ -0,0 +1,27 @@ +use crate::planner_fixtures::models::{FixtureColumn, FixtureStarRelation}; +use crate::relation_names::models::RelationName; + +/// A relation written `schema.name`, or `name` without a schema. +pub(super) fn qualified(relation: &RelationName) -> String { + let parts: Vec<&str> = relation + .schema + .iter() + .map(String::as_str) + .chain([relation.name.as_str()]) + .collect(); + parts.join(".") +} + +/// A fixture relation `name` with one typed column. +pub(super) fn fixture_relation(name: &str) -> FixtureStarRelation { + FixtureStarRelation { + key: ("model".to_owned(), name.to_owned()), + columns: vec![FixtureColumn { + name: "id".to_owned(), + folded_name: "id".to_owned(), + sql_type: Some("Int32".to_owned()), + nullable: Some(false), + }], + authoritative: true, + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_dialect.rs index 09c381deb5..a4c9f9917d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_clickhouse_dialect.rs @@ -138,13 +138,13 @@ fn given_clickhouse_model_sql_when_probing_the_native_parser_then_only_unreadabl expected_gap: None, }, NativeGapTestCase { - description: "INTO OUTFILE is a named gap at its line and column", - sql: "SELECT 'é' AS a\nFROM t\n INTO OUTFILE 'x.csv'", + description: "WITH TIES is a named gap at its line and column", + sql: "SELECT 'é' AS a\nFROM t\n ORDER BY a LIMIT 1 WITH TIES", placeholders: &[], expected_gap: Some(LocatedGap { - construct: "into_outfile".to_owned(), + construct: "with_ties".to_owned(), line: 3, - column: 3, + column: 22, }), }, NativeGapTestCase { diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs new file mode 100644 index 0000000000..02d8b686a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs @@ -0,0 +1,249 @@ +use std::collections::BTreeMap; + +use crate::analysis_route::tests::helpers::{fixture_relation, qualified}; +use crate::analysis_route::tests::test_types::{ + FingerprintTestCase, QueryShapeTestCase, RawShapeTestCase, RelationNamesTestCase, + StarKeysTestCase, +}; +use crate::diff_selection::main::raw_query_shape::raw_query_shape; +use crate::planner_fixtures::main::star_keys::star_keys; +use crate::planner_migrations::main::migration_fingerprint::migration_fingerprint; +use crate::planner_migrations::main::query_shape::query_shape; +use crate::relation_names::main::extract_relation_names::extract_relation_names; + +/// A ClickHouse query the native parser reads. +const READABLE: &str = "SELECT o.id, o.amount AS total FROM orders AS o"; +/// A ClickHouse query the native parser stops at with a named gap. +const UNREADABLE: &str = "SELECT o.id FROM orders AS o ORDER BY o.id LIMIT 1 WITH TIES"; +const DIALECT: &str = "clickhouse"; + +#[test] +fn given_clickhouse_sql_when_proving_a_column_migration_then_only_sql_read_whole_has_a_shape() { + let test_cases = [ + QueryShapeTestCase { + description: "SQL read whole has its projections", + sql: READABLE, + expected_projections: Some(&["id", "total"]), + }, + QueryShapeTestCase { + description: "SQL with a gap has no shape", + sql: UNREADABLE, + expected_projections: None, + }, + ]; + for test_case in test_cases { + let shape = query_shape(test_case.sql, DIALECT).expect("no internal failure"); + let projections: Option> = shape + .as_ref() + .map(|shape| shape.projections.iter().map(|p| p.name.as_str()).collect()); + assert_eq!( + projections.as_deref(), + test_case.expected_projections, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_clickhouse_sql_when_fingerprinting_a_migration_then_formatting_is_ignored_and_gaps_have_none() + { + let test_cases = [ + FingerprintTestCase { + description: "the same SQL has the same fingerprint", + sql: READABLE, + expected_same: Some(true), + }, + FingerprintTestCase { + description: "layout and comments are ignored", + sql: "SELECT o.id,\n o.amount AS total -- note\nFROM orders AS o", + expected_same: Some(true), + }, + FingerprintTestCase { + description: "a renamed column changes the fingerprint", + sql: "SELECT o.id, o.amount AS sum FROM orders AS o", + expected_same: Some(false), + }, + FingerprintTestCase { + description: "SQL with a gap has no fingerprint", + sql: UNREADABLE, + expected_same: None, + }, + ]; + let identities: BTreeMap = BTreeMap::new(); + let base = migration_fingerprint(READABLE, "", &identities, DIALECT) + .expect("no internal failure") + .expect("readable SQL has a fingerprint"); + for test_case in test_cases { + let same: Option = migration_fingerprint(test_case.sql, "", &identities, DIALECT) + .expect("no internal failure") + .map(|fingerprint| fingerprint == base); + assert_eq!(same, test_case.expected_same, "{}", test_case.description); + } +} + +#[test] +fn given_a_clickhouse_diff_query_when_reading_its_shape_then_unreadable_sql_is_refused() { + let test_cases = [ + RawShapeTestCase { + description: "a SELECT read whole is a read-only select", + sql: READABLE, + expected_shape: Some(("select", false)), + }, + RawShapeTestCase { + description: "INTO OUTFILE writes", + sql: "SELECT o.id FROM orders AS o INTO OUTFILE 'x.csv'", + expected_shape: Some(("select", true)), + }, + RawShapeTestCase { + description: "SQL with a gap is refused", + sql: UNREADABLE, + expected_shape: None, + }, + ]; + for test_case in test_cases { + let read = raw_query_shape(test_case.sql, DIALECT); + let shape: Option<(&str, bool)> = read + .iter() + .map(|shape| (shape.root_key.as_str(), shape.writes_or_locks)) + .next(); + assert_eq!(shape, test_case.expected_shape, "{}", test_case.description); + } +} + +#[test] +fn given_clickhouse_sql_when_extracting_relation_names_then_they_are_polyglots() { + let orders: &[&str] = &["analytics.orders"]; + let test_cases = [ + RelationNamesTestCase { + description: "a CTE is not a relation", + sql: "WITH c AS (SELECT 1 AS id) SELECT * FROM shop.orders AS o JOIN c ON o.id = c.id", + expected_names: Some((&["shop.orders"], &[])), + }, + RelationNamesTestCase { + description: "SQL with a gap is not read", + sql: UNREADABLE, + expected_names: None, + }, + RelationNamesTestCase { + description: "TRUNCATE names its table", + sql: "TRUNCATE TABLE analytics.orders", + expected_names: Some((orders, &[])), + }, + RelationNamesTestCase { + description: "ALTER ... DELETE names its table", + sql: "ALTER TABLE analytics.orders DELETE WHERE orders_id = 1", + expected_names: Some((orders, &[])), + }, + RelationNamesTestCase { + description: "INTO OUTFILE names the table read", + sql: "SELECT * FROM analytics.orders INTO OUTFILE 'x.csv' AND STDOUT", + expected_names: Some((orders, &[])), + }, + RelationNamesTestCase { + description: "a temporary table is named and kept as temporary", + sql: "CREATE TEMPORARY TABLE scratch AS SELECT * FROM analytics.orders", + expected_names: Some((&["analytics.orders", "scratch"], &["scratch"])), + }, + RelationNamesTestCase { + description: "OPTIMIZE is a command naming no relation, as Polyglot reads it", + sql: "OPTIMIZE TABLE analytics.orders FINAL", + expected_names: Some((&[], &[])), + }, + RelationNamesTestCase { + description: "an ALTER with an UPDATE is a command naming no relation, as Polyglot reads it", + sql: "ALTER TABLE analytics.orders UPDATE status = 'x' WHERE 1", + expected_names: Some((&[], &[])), + }, + RelationNamesTestCase { + description: "a form the parser still gaps is not read", + sql: "TRUNCATE TABLE analytics.orders SETTINGS a = 1", + expected_names: None, + }, + ]; + for test_case in test_cases { + let found = extract_relation_names(test_case.sql, Some(DIALECT)).expect("no failure"); + let relations: Option> = found + .as_ref() + .map(|found| found.relations.iter().map(qualified).collect()); + let names: Option<(Vec<&str>, Vec<&str>)> = + relations + .as_ref() + .zip(found.as_ref()) + .map(|(relations, found)| { + ( + relations.iter().map(String::as_str).collect(), + found.temporary.iter().map(String::as_str).collect(), + ) + }); + assert_eq!( + names + .as_ref() + .map(|(relations, temporary)| (relations.as_slice(), temporary.as_slice())), + test_case.expected_names, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_clickhouse_star_when_selecting_fixture_dependencies_then_unsettled_stars_take_every_relation() + { + let every: &[&str] = &["orders", "customers"]; + let test_cases = [ + StarKeysTestCase { + description: "a qualified star takes its relation", + sql: "SELECT o.* FROM orders AS o JOIN customers AS c ON o.id = c.id", + expected_relations: &["orders"], + }, + StarKeysTestCase { + description: "no star takes no relation", + sql: "SELECT count(*) AS n FROM orders AS o JOIN customers AS c ON o.id = c.id", + expected_relations: &[], + }, + StarKeysTestCase { + description: "a bare star takes every relation in the query", + sql: "SELECT * FROM orders AS o JOIN customers AS c ON o.id = c.id", + expected_relations: every, + }, + StarKeysTestCase { + description: "an unknown qualifier takes every relation", + sql: "SELECT zz.* FROM orders AS o JOIN customers AS c ON o.id = c.id", + expected_relations: every, + }, + StarKeysTestCase { + description: "an alias reused across set operation branches takes both relations", + sql: "SELECT t.* FROM orders t UNION ALL SELECT t.* FROM customers t", + expected_relations: every, + }, + StarKeysTestCase { + description: "an alias reused in a subquery takes both relations", + sql: "SELECT o.* FROM orders o WHERE o.id IN (SELECT o.id FROM customers o)", + expected_relations: every, + }, + StarKeysTestCase { + description: "an alias reused across CTEs takes both relations", + sql: "WITH a AS (SELECT x.* FROM orders x), b AS (SELECT x.* FROM customers x) \ + SELECT a.id FROM a JOIN b ON a.id = b.id", + expected_relations: every, + }, + StarKeysTestCase { + description: "SQL with a gap takes every relation", + sql: "SELECT o.* FROM orders AS o ORDER BY o.id LIMIT 1 WITH TIES", + expected_relations: every, + }, + ]; + let relations = [fixture_relation("orders"), fixture_relation("customers")]; + for test_case in test_cases { + let keys: Vec = star_keys(test_case.sql, &relations, Some(DIALECT), None) + .into_iter() + .map(|(_, name)| name) + .collect(); + assert_eq!( + keys, test_case.expected_relations, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 43cd5cec87..d7b1470566 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,7 @@ use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; +use sqlquack_syntax::expressions::names::models::Null; +use sqlquack_syntax::{Expression, Parsed}; pub(super) struct NativeGapTestCase { pub(super) description: &'static str, @@ -31,7 +33,7 @@ impl NativeDialect for PivotlessDialect { "postgres" } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { + fn parse(&self, sql: &str) -> Result { let refused = ["PIVOT", "@@@", "__sqlbuild_udf_", "__cursor_", "@'"] .iter() .filter_map(|word| sql.find(word).map(|offset| (offset, *word))) @@ -41,7 +43,10 @@ impl NativeDialect for PivotlessDialect { construct: word.to_owned(), offset, }), - None => Ok(()), + None => Ok(Parsed { + statements: vec![Expression::Null(Null)], + gaps: Vec::new(), + }), } } @@ -71,3 +76,36 @@ pub(super) struct TypeSpellingsTestCase { pub(super) spellings: &'static [&'static str], pub(super) expected_mismatches: &'static [&'static str], } + +pub(super) struct QueryShapeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_projections: Option<&'static [&'static str]>, +} + +pub(super) struct FingerprintTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Whether the fingerprint equals `READABLE`'s, or None for no fingerprint. + pub(super) expected_same: Option, +} + +pub(super) struct RawShapeTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// The root kind and whether it writes or locks, or None for a refusal. + pub(super) expected_shape: Option<(&'static str, bool)>, +} + +pub(super) struct RelationNamesTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// The qualified relations and temporary tables, or None when the SQL is not read. + pub(super) expected_names: Option<(&'static [&'static str], &'static [&'static str])>, +} + +pub(super) struct StarKeysTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_relations: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index fedc80ad24..9accb9c990 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,5 +1,8 @@ //! What a dialect's native analysis module provides. +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::constants::{STATEMENTS_CONSTRUCT, UNPARSED_STATEMENT_CONSTRUCT}; use crate::analysis_route::models::ParserGap; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. @@ -7,8 +10,30 @@ pub trait NativeDialect: Sync { /// The adapter's analysis dialect name, such as `postgres`. fn name(&self) -> &'static str; - /// Whether the native parser reads `sql` whole: one statement, no gap and no unparsed bytes. - fn read_whole(&self, sql: &str) -> Result<(), ParserGap>; + /// Every statement of `sql` with the parser's named gaps, or the first error. + fn parse(&self, sql: &str) -> Result; + + /// The one statement of `sql`, with no gap and not a lone unparsed command. + fn read_whole(&self, sql: &str) -> Result { + let mut parsed: Parsed = self.parse(sql)?; + if let Some(gap) = parsed.gaps.first() { + return Err(ParserGap { + construct: gap.construct.to_owned(), + offset: gap.start, + }); + } + let unread = |construct: &str| ParserGap { + construct: construct.to_owned(), + offset: sql.len(), + }; + if parsed.statements.len() != 1 { + return Err(unread(STATEMENTS_CONSTRUCT)); + } + match parsed.statements.remove(0) { + Expression::Command(_) => Err(unread(UNPARSED_STATEMENT_CONSTRUCT)), + statement => Ok(statement), + } + } /// The engine-accepted prefix of UDF and cursor stubs: `__sqlbuild_`, or `__sqlbuild.` for T-SQL. fn stub_prefix(&self) -> &'static str; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs index c553e213e1..81cd6076ec 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -1,14 +1,53 @@ -//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. +//! A raw diff query's shape, parsed by SQLQuack where native, else by guarded Polyglot. use polyglot_sql::dialects::Dialect; use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; +use sqlquack_syntax::traversal::types::ExpressionWalk as NativeWalk; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; use crate::diff_selection::models::RawQueryShape; -/// The query's shape, or the text of the exception the wheel raises for it. +/// The query's shape, or why it is not one statement the dialect's parser reads whole. pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { + match analysis_route(Some(dialect)) { + AnalysisRoute::Native(native) => { + let root: sqlquack_syntax::Expression = native + .read_whole(sql) + .map_err(|gap| format!("the {dialect} parser cannot read {}", gap.construct))?; + native_shape(&root) + } + AnalysisRoute::Legacy => polyglot_shape(sql, dialect), + } +} + +/// The shape of a statement SQLQuack read whole. +fn native_shape(root: &sqlquack_syntax::Expression) -> Result { + let root_key: String = root.variant_name().to_lowercase(); + let node_keys: Vec = if RAW_QUERY_READ_ONLY_ROOTS.contains(&root_key.as_str()) { + NativeWalk::dfs(root) + .map(|node| node.variant_name().to_lowercase()) + .collect() + } else { + Vec::new() + }; + let writes_or_locks: bool = match root { + sqlquack_syntax::Expression::Select(select) => { + select.into.is_some() || !select.locks.is_empty() + } + _ => false, + }; + Ok(RawQueryShape { + root_key, + node_keys, + writes_or_locks, + }) +} + +/// The shape Polyglot reads, or the text of the exception the wheel raises. +fn polyglot_shape(sql: &str, dialect: &str) -> Result { let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; let guard: ComplexityGuardOptions = diff --git a/crates/sqlbuild-analysis/src/explicit_references/tests/helpers.rs b/crates/sqlbuild-analysis/src/explicit_references/tests/helpers.rs index 7ee8a76c62..7e4865cbf8 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/tests/helpers.rs @@ -1,5 +1,6 @@ use crate::explicit_references::models::{ - ModelReferences, NamedRelation, ReferenceRequest, Relation, SourceRelation, + LiteralSql, ModelReferences, NamedRelation, ReferenceRequest, Relation, SourceRelation, + SqlOwner, }; pub(crate) fn relation(name: &str, schema: Option<&str>, database: Option<&str>) -> Relation { @@ -65,3 +66,28 @@ pub(crate) fn request() -> ReferenceRequest { owners: Vec::new(), } } + +/// `request()` for ClickHouse with one node running `statements` as literal SQL, in line order. +pub(crate) fn clickhouse_request(statements: &[&str]) -> ReferenceRequest { + let literals: Vec = statements + .iter() + .enumerate() + .map(|(index, sql)| LiteralSql { + method: "execute_sql".to_owned(), + sql: (*sql).to_owned(), + line: index + 1, + }) + .collect(); + ReferenceRequest { + dialect: Some("clickhouse".to_owned()), + owners: vec![SqlOwner { + label: "task 'export_orders'".to_owned(), + kind: "node".to_owned(), + path_order: 0, + own_refs: Vec::new(), + upstream_loaders: Vec::new(), + literals, + }], + ..request() + } +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/tests/test_explicit_references.rs b/crates/sqlbuild-analysis/src/explicit_references/tests/test_explicit_references.rs index acbddce0e6..3dc34677c0 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/tests/test_explicit_references.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/tests/test_explicit_references.rs @@ -1,6 +1,7 @@ use crate::explicit_references::_helpers::relations::{matched, project_relations}; -use crate::explicit_references::tests::helpers::{relation, request}; -use crate::explicit_references::tests::test_types::MatchTestCase; +use crate::explicit_references::main::check_explicit_references::check_explicit_references; +use crate::explicit_references::tests::helpers::{clickhouse_request, relation, request}; +use crate::explicit_references::tests::test_types::{HardCodedSqlTestCase, MatchTestCase}; #[test] fn given_written_relations_when_matching_project_relations_then_matches_python() { @@ -60,3 +61,57 @@ fn given_written_relations_when_matching_project_relations_then_matches_python() assert_eq!(found, test_case.expected_match, "{}", test_case.description); } } + +#[test] +fn given_clickhouse_python_sql_when_checking_hard_coded_names_then_each_statement_reads_as_polyglot_read_it() + { + let test_cases = [ + HardCodedSqlTestCase { + description: "TRUNCATE names a model", + statements: &["TRUNCATE TABLE main.orders"], + expected_p008_count: 1, + }, + HardCodedSqlTestCase { + description: "ALTER ... DELETE names a model", + statements: &["ALTER TABLE main.orders DELETE WHERE id = 1"], + expected_p008_count: 1, + }, + HardCodedSqlTestCase { + description: "a SELECT ... INTO OUTFILE names a model", + statements: &["SELECT * FROM main.customers INTO OUTFILE 'x.csv'"], + expected_p008_count: 1, + }, + HardCodedSqlTestCase { + description: "a temporary table hides the model of the same name", + statements: &[ + "CREATE TEMPORARY TABLE customers AS SELECT 1 AS id", + "SELECT * FROM customers", + ], + expected_p008_count: 0, + }, + HardCodedSqlTestCase { + description: "OPTIMIZE names no relation, as Polyglot reads it", + statements: &["OPTIMIZE TABLE main.orders FINAL"], + expected_p008_count: 0, + }, + HardCodedSqlTestCase { + description: "an ALTER with an UPDATE names no relation, as Polyglot reads it", + statements: &["ALTER TABLE main.orders UPDATE a = 1 WHERE 1"], + expected_p008_count: 0, + }, + ]; + for test_case in test_cases { + let report = + check_explicit_references(&clickhouse_request(test_case.statements)).expect("checked"); + let found: usize = report + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.code == "P008") + .count(); + assert_eq!( + found, test_case.expected_p008_count, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/tests/test_types.rs b/crates/sqlbuild-analysis/src/explicit_references/tests/test_types.rs index 3d4be469e1..ecc354f2f5 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/tests/test_types.rs @@ -5,3 +5,10 @@ pub(crate) struct MatchTestCase { /// The matched `(kind, name, compatibility_for)`, or None. pub(crate) expected_match: Option<(&'static str, &'static str, Option<&'static str>)>, } + +pub(crate) struct HardCodedSqlTestCase { + pub(crate) description: &'static str, + /// Literal SQL a ClickHouse node runs, in line order. + pub(crate) statements: &'static [&'static str], + pub(crate) expected_p008_count: usize, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs index 9c1c3c67c6..502ddf9d47 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod native_star_keys; pub(crate) mod resolved_reads; pub(crate) mod schema_order; pub(crate) mod stub_markers; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/native_star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/native_star_keys.rs new file mode 100644 index 0000000000..f0a2f5efd8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/native_star_keys.rs @@ -0,0 +1,99 @@ +//! Star dependencies read from SQLQuack's tree, a superset of what each star reads. + +use std::collections::{HashMap, HashSet}; + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::Expression; +use sqlquack_syntax::expressions::queries::models::Select; +use sqlquack_syntax::traversal::types::ExpressionWalk; + +use crate::analysis_route::types::NativeDialect; +use crate::planner_fixtures::_helpers::stub_markers::stub_markers; +use crate::planner_fixtures::models::FixtureStarRelation; + +/// Each qualifier in a query, casefolded, to every relation it names there. +type Qualifiers = HashMap>; + +/// Relations the stars read: the qualifier's relations, or every relation given when unsettled. +pub(crate) fn native_star_keys( + native: &dyn NativeDialect, + sql: &str, + relations: &[FixtureStarRelation], +) -> Vec<(String, String)> { + let every = || { + relations + .iter() + .map(|relation| relation.key.clone()) + .collect() + }; + let Ok(sql) = stub_markers(sql) else { + return every(); + }; + let Ok(root) = native.read_whole(&sql) else { + return every(); + }; + let Some(names) = star_relations(&root) else { + return every(); + }; + relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect() +} + +/// The relations the query's stars read, or None when a star's qualifier names no relation. +fn star_relations(root: &Expression) -> Option> { + let mut qualifiers: Qualifiers = HashMap::new(); + let mut stars: Vec> = Vec::new(); + for node in ExpressionWalk::dfs(root) { + match node { + Expression::Table(table) => { + let name: String = python_casefold(&table.name.name); + let written = std::iter::once(name.clone()) + .chain(table.alias.iter().map(|alias| python_casefold(&alias.name))); + for qualifier in written { + qualifiers + .entry(qualifier) + .or_default() + .insert(name.clone()); + } + } + Expression::Select(select) => stars.extend(projected_stars(select)), + _ => {} + } + } + let mut names: HashSet = HashSet::new(); + for star in &stars { + let read: &HashSet = match star { + Some(qualifier) => qualifiers.get(qualifier)?, + None => { + names.extend(qualifiers.values().flatten().cloned()); + continue; + } + }; + names.extend(read.iter().cloned()); + } + Some(names) +} + +/// The SELECT's projected stars: each one's casefolded qualifier, or None for a bare star. +fn projected_stars(select: &Select) -> Vec> { + select + .expressions + .iter() + .filter_map(star_qualifier) + .collect() +} + +/// A star projection's casefolded qualifier, None for a bare star; not a star, no entry. +fn star_qualifier(projection: &Expression) -> Option> { + let Expression::Star(star) = projection else { + return None; + }; + Some( + star.table + .as_ref() + .map(|table| python_casefold(&table.name)), + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs index 5afda5f19d..87631c9371 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -8,6 +8,9 @@ use polyglot_sql::{ }; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::planner_fixtures::_helpers::native_star_keys::native_star_keys; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; use crate::planner_fixtures::models::FixtureStarRelation; @@ -32,6 +35,9 @@ pub fn star_keys( { return Vec::new(); } + if let AnalysisRoute::Native(native) = analysis_route(dialect) { + return native_star_keys(native, sql, relations); + } let Ok(dialect) = DialectType::from_str(dialect.filter(|name| !name.is_empty()).unwrap_or("generic")) else { diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs index 338624014a..7edba4a616 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs @@ -1,11 +1,14 @@ //! Normalize migration metadata and SQL without changing the persisted format. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::planner_changes::main::version_identity_payload::version_identity_payload; use crate::planner_changes::models::IdentityPayload; use crate::planner_migrations::_helpers::proofs::ast::strip_formatting; +use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::_helpers::proofs::local_names::{local_names, rewrite, unambiguous}; -use crate::planner_migrations::_helpers::proofs::parse::parse; -use crate::planner_migrations::constants::{EXCLUDED_CONFIG, REF_MARKER}; +use crate::planner_migrations::_helpers::proofs::parse::proof_tree; +use crate::planner_migrations::constants::{EXCLUDED_CONFIG, NATIVE_TREE_PREFIX, REF_MARKER}; use polyglot_sql::{Expression, dialects::Dialect}; use regex::Regex; use serde_json::Value; @@ -52,7 +55,7 @@ pub(crate) fn normalized( let digest = format!("{:x}", Sha256::digest(identity.as_bytes())); format!("__sqb_migration_ref__{}", &digest[..16]) }); - let Some(mut parsed) = parse(&placeholder, dialect)? else { + let Some(mut parsed) = proof_tree(&placeholder, dialect)? else { return Ok(None); }; strip_formatting(&mut parsed); @@ -74,6 +77,9 @@ pub(crate) fn normalized( } } rewrite(&mut parsed, &names, &ctes); + if let AnalysisRoute::Native(_) = analysis_route(Some(dialect)) { + return encode(&parsed).map(|tree| Some(format!("{NATIVE_TREE_PREFIX}{tree}"))); + } let expression: Expression = serde_json::from_value(parsed).map_err(|error| error.to_string())?; Dialect::get_by_name(dialect) diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs index a84b3ee534..e344bd397a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs @@ -1,6 +1,9 @@ -//! Parse migration SQL under the compiler's complexity limit. +//! Parse migration SQL with the dialect's parser: SQLQuack where native, else guarded Polyglot. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::planner_migrations::constants::{SET_OPERATION_LEFT, WITH_KEY}; use polyglot_sql::dialects::Dialect; use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; use serde_json::{Value, json}; @@ -24,3 +27,49 @@ pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { .map(Some) .map_err(|error| error.to_string()) } + +/// The one statement of `sql` as a tree; None, the conservative plan, if not read whole. +pub(crate) fn proof_tree(sql: &str, dialect: &str) -> Result, String> { + let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) else { + return parse(sql, dialect); + }; + let Ok(statement) = native.read_whole(sql) else { + return Ok(None); + }; + let mut tree: Value = serde_json::to_value(statement).map_err(|error| error.to_string())?; + hoist_leading_with(&mut tree); + Ok(Some(tree)) +} + +/// Move a set operation's leading WITH from its first SELECT onto the set operation. +fn hoist_leading_with(tree: &mut Value) { + let Some(operation) = tree + .as_object_mut() + .and_then(|object| object.values_mut().next()) + .filter(|operation| operation.get(SET_OPERATION_LEFT).is_some()) + else { + return; + }; + if !operation[WITH_KEY].is_null() { + return; + } + let with: Value = leading_with(&mut operation[SET_OPERATION_LEFT]); + operation[WITH_KEY] = with; +} + +/// The WITH of the first SELECT under `node`, taken out of it; null when it has none. +fn leading_with(node: &mut Value) -> Value { + let Some(inner) = node + .as_object_mut() + .and_then(|object| object.values_mut().next()) + else { + return Value::Null; + }; + if inner.get(SET_OPERATION_LEFT).is_some() { + return leading_with(&mut inner[SET_OPERATION_LEFT]); + } + inner + .as_object_mut() + .and_then(|select| select.insert(WITH_KEY.to_owned(), Value::Null)) + .unwrap_or(Value::Null) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs index b760634f78..1cd92ada6a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -24,6 +24,8 @@ pub(crate) const OPAQUE_TUPLE_FIELDS: &[&str] = &[ ]; pub(crate) const HAVING_MAX_KEY: &str = "having_max"; pub(crate) const WITH_KEY: &str = "with"; +/// A set operation's first branch. +pub(crate) const SET_OPERATION_LEFT: &str = "left"; pub(crate) const TABLE_KEY: &str = "table"; pub(crate) const NUMBER_LITERAL: &str = "number"; @@ -49,3 +51,5 @@ pub(crate) const POLICY_FAIL: &str = "fail"; pub(crate) const MODEL_DONE: &str = "done"; pub(crate) const MODEL_RENAMED: &str = "renamed"; pub(crate) const MODEL_CONFLICT: &str = "conflict"; +/// Marks a native dialect's normalized SQL, its canonical tree, so it never equals Polyglot's SQL text. +pub(crate) const NATIVE_TREE_PREFIX: &str = "sqlquack-tree:"; diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs index c91df4b1fb..0a2fee54e1 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/query_shape.rs @@ -4,14 +4,14 @@ use crate::planner_migrations::_helpers::proofs::ast::{canonical, strip_formatti use crate::planner_migrations::_helpers::proofs::clauses::{clause_references, positional}; use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::_helpers::proofs::inputs::select_inputs; -use crate::planner_migrations::_helpers::proofs::parse::parse; +use crate::planner_migrations::_helpers::proofs::parse::proof_tree; use crate::planner_migrations::_helpers::proofs::projection::named_projections; use crate::planner_migrations::models::QueryShape; use serde_json::json; /// Named projections of exactly one plain SELECT, or None when unknowable. pub fn query_shape(sql: &str, dialect: &str) -> Result, String> { - let Some(mut root) = parse(sql, dialect)? else { + let Some(mut root) = proof_tree(sql, dialect)? else { return Ok(None); }; strip_formatting(&mut root); diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs index aae5b546d0..9294a01917 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs @@ -1,8 +1,12 @@ -//! Cursor types of warehouse type names and Polyglot type arguments. +//! Cursor types of warehouse type names and parsed type arguments. use serde_json::{Map, Value}; +use sqlquack_syntax::traversal::types::ExpressionWalk; +use sqlquack_syntax::{Expression, ParseOptions}; -use crate::planner_strategy::constants::{INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES}; +use crate::planner_strategy::constants::{ + INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES, TYPE_PARSE_DIALECT, +}; use crate::planner_strategy::models::CursorType; /// The cursor type of a known type name. @@ -25,3 +29,28 @@ pub(crate) fn argument(args: &Map, key: &str) -> String { Some(other) => other.to_string().to_uppercase(), } } + +/// The scalar arguments of `type_sql` as SQLQuack's parser reads it in a cast, or why not. +pub(crate) fn type_args(type_sql: &str) -> Result, String> { + let statements: Vec = sqlquack_syntax::parse_strict( + &format!("SELECT CAST(NULL AS {type_sql})"), + TYPE_PARSE_DIALECT, + &ParseOptions::default(), + ) + .map_err(|error| error.to_string())?; + let data_type = statements + .iter() + .flat_map(|statement| statement.dfs()) + .find_map(|node| match node { + Expression::Cast(cast) => Some(&cast.to), + _ => None, + }) + .ok_or_else(|| format!("not one type: {type_sql}"))?; + match serde_json::to_value(data_type).map_err(|error| error.to_string())? { + Value::Object(fields) => Ok(fields + .into_iter() + .filter(|(_, value)| !matches!(value, Value::Object(_) | Value::Array(_))) + .collect()), + other => Err(format!("not one type: {other}")), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 1176dc46a1..ad15667c5d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -72,10 +72,10 @@ pub const INTEGER_TYPE_NAMES: [&str; 17] = [ ]; /// Decimal type names whose zero-scale form is an integer cursor. pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; -/// The type Polyglot reports for a type it does not know. -pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; -/// The dialect cursor type classification parses unknown types with. -pub const GENERIC_DIALECT: &str = "generic"; +/// The type the parser reports for a type it does not know. +pub const CUSTOM_TYPE_NAME: &str = "CUSTOM"; +/// The parser cursor type classification reads an unknown warehouse type with. +pub(crate) const TYPE_PARSE_DIALECT: sqlquack_syntax::Dialect = sqlquack_syntax::Dialect::DuckDB; /// Error code of an authored cursor input column its input does not expose. pub const CURSOR_INPUT_COLUMN_ERROR_CODE: &str = "S302"; /// Incremental strategies whose microbatches replay one batch by default. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs index 2be7a8dd5b..f0b4a53575 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs @@ -84,7 +84,7 @@ impl CappedWatermarkInput { } } -/// A warehouse type Polyglot could not parse, which classifies as no cursor type. +/// A warehouse type the parser could not read, which classifies as no cursor type. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TypeParseFailure { pub normalized_type: String, diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs index 22dedf16bd..4853f55f17 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs @@ -3,13 +3,12 @@ use serde_json::{Map, Value}; use crate::planner_strategy::_helpers::decimal_scale::decimal_scale_is_zero; -use crate::planner_strategy::_helpers::type_names::{argument, named_type}; -use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME}; +use crate::planner_strategy::_helpers::type_names::{argument, named_type, type_args}; +use crate::planner_strategy::constants::CUSTOM_TYPE_NAME; use crate::planner_strategy::errors::TypeParseFailure; use crate::planner_strategy::models::CursorType; -use crate::type_system::main::polyglot_type_args::polyglot_type_args; -/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type. +/// Timestamp or integer for supported cross-adapter types; `Err` when the parser cannot read an unknown type. pub fn classify_cursor_sql_type(sql_type: &str) -> Result, TypeParseFailure> { let normalized: String = sql_type .to_uppercase() @@ -28,13 +27,12 @@ pub fn classify_cursor_sql_type(sql_type: &str) -> Result, Ty if let Some(cursor_type) = named_type(&base) { return Ok(Some(cursor_type)); } - let args: Map = - polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure { - normalized_type: normalized.clone(), - message, - })?; + let args: Map = type_args(&normalized).map_err(|message| TypeParseFailure { + normalized_type: normalized.clone(), + message, + })?; let mut type_name: String = argument(&args, "data_type"); - if type_name == POLYGLOT_CUSTOM_TYPE_NAME { + if type_name == CUSTOM_TYPE_NAME { type_name = argument(&args, "name").replace('_', ""); } Ok(named_type(&type_name)) diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs index d7b05b6719..9293db1051 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs @@ -11,7 +11,7 @@ fn cursor_type_name(cursor_type: CursorType) -> &'static str { } } -/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any Polyglot parse error that left it unclassified. +/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any parse error that left it unclassified. pub fn cursor_type_warning( check: &CursorTypeCheck, ) -> (Option, Option) { diff --git a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs index 28fe14df79..277325f0fb 100644 --- a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs @@ -4,6 +4,8 @@ use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; use serde_json::{Map, Value, json}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::relation_names::constants::{ CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, NAME_KEY, QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, @@ -19,14 +21,12 @@ pub(crate) fn extracted_relations( let dialect: &str = dialect .filter(|name| !name.is_empty()) .unwrap_or(GENERIC_DIALECT); - let Some(statements) = parsed_statements(sql, dialect)? else { + let Some(trees) = statement_trees(sql, dialect)? else { return Ok(None); }; let mut extracted: ExtractedRelations = ExtractedRelations::default(); - for statement in &statements { - let tree: Value = serde_json::to_value(statement) - .map_err(|error| RelationNamesFailure::Internal(error.to_string()))?; - let nodes: Vec<&Map> = walked(&tree); + for tree in &trees { + let nodes: Vec<&Map> = walked(tree); let cte_names: Vec = nodes.iter().flat_map(|node| cte_names(node)).collect(); extracted .temporary @@ -45,6 +45,35 @@ pub(crate) fn extracted_relations( Ok(Some(extracted)) } +/// Each statement as a tree: SQLQuack's where native, with any gap or error None; else Polyglot's. +fn statement_trees(sql: &str, dialect: &str) -> Result>, RelationNamesFailure> { + let internal = |error: serde_json::Error| RelationNamesFailure::Internal(error.to_string()); + if let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) { + if sql.trim().is_empty() { + return Ok(Some(Vec::new())); + } + return match native.parse(sql) { + Ok(parsed) if parsed.gaps.is_empty() => parsed + .statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal), + _ => Ok(None), + }; + } + let Some(statements) = parsed_statements(sql, dialect)? else { + return Ok(None); + }; + statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal) +} + /// The wheel's guarded `parse`: Ok(None) is the `PolyglotError` Python catches. fn parsed_statements( sql: &str, diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs index 7e49d9c53d..205c4563f7 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs @@ -2,7 +2,11 @@ use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; -use crate::snowflake_cost::constants::{CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER}; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::snowflake_cost::constants::{ + CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER, SNOWFLAKE_DIALECT, +}; pub(crate) fn of_kind(query_type: &str, kind: &str) -> bool { query_type == kind @@ -16,8 +20,13 @@ pub(crate) fn python_space(character: char) -> bool { character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) } -/// Whether a CREATE statement's text parses as a definition that needs no warehouse. +/// Whether a CREATE needs no warehouse; false, warehouse-billed, when not read whole. pub(crate) fn metadata_only_create(query_text: &str) -> bool { + if let AnalysisRoute::Native(native) = analysis_route(Some(SNOWFLAKE_DIALECT)) { + return native + .read_whole(query_text) + .is_ok_and(|statement| native_metadata_only(&statement)); + } let options: ParseOptions = ParseOptions { complexity_guard: None, }; @@ -41,3 +50,18 @@ pub(crate) fn metadata_only_create(query_text: &str) -> bool { statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), } } + +/// Whether a statement SQLQuack read is a CREATE that needs no warehouse. +fn native_metadata_only(statement: &sqlquack_syntax::Expression) -> bool { + match statement { + sqlquack_syntax::Expression::CreateTable(table) => { + !(table.as_select.is_some() + || table.clone_source.is_some() + || table.using_template.is_some() + || table.table_modifier.as_deref().unwrap_or("").to_uppercase() + == DYNAMIC_TABLE_MODIFIER) + } + sqlquack_syntax::Expression::CreateView(view) if view.materialized => false, + statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs index 67a32cb1f1..9eae6ee8d0 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -8,3 +8,5 @@ pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; pub const QUERY_TAG_APP: &str = "sqlbuild"; pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; +/// Snowflake's analysis dialect name, which chooses its parser. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs index 1ac950cb8f..4611bff368 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -1,4 +1,3 @@ pub mod normalize_text_type; pub mod normalize_type; -pub mod polyglot_type_args; pub mod same_type; diff --git a/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs b/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs deleted file mode 100644 index 972ac1f6c0..0000000000 --- a/crates/sqlbuild-analysis/src/type_system/main/polyglot_type_args.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! The scalar arguments of a type as Polyglot parses it. - -use polyglot_sql::{DataType, Dialect, DialectType}; -use serde_json::{Map, Value}; - -use crate::type_system::_helpers::parsed::parse_type; -use crate::type_system::_helpers::shallow_args::shallow_args; - -/// `parse_data_type(type_sql, dialect=...).args` scalars, or the parse error text. -pub fn polyglot_type_args(type_sql: &str, dialect: &str) -> Result, String> { - let polyglot: Dialect = dialect - .parse::() - .map(Dialect::get) - .map_err(|_| format!("unknown dialect '{dialect}'"))?; - let parsed: DataType = parse_type(type_sql, &polyglot)?; - Ok(shallow_args(&parsed)) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index 48f55a3095..840b93dcea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::records::ids::RecordId; const PLANNER_LOGGER: &str = "sqlbuild.planner"; -/// Log the Polyglot parse error that left a warehouse type unclassified. +/// Log the parse error that left a warehouse type unclassified. fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { let text = |value: &str| PyString::new(py, value).into_any(); call_with( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index bf24e9cfa0..5b94a917f1 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -109,8 +109,8 @@ class SlowConnectionDuckDbAdapter(DuckDbAdapter): """An in-memory DuckDB adapter whose connections, compile profile and rules dialect take `connection_seconds`, `compile_seconds` and `rules_seconds` on a manual clock. - The compile profile takes its time on its first call; the dialect on its second, which the - rules gate makes after the compile's own dialect read.""" + The compile profile takes its time on its first call; the dialect on its third, which the + rules gate makes after the compile's two dialect reads (its adapter context and its profile).""" def __init__( self, @@ -124,7 +124,7 @@ def __init__( self._clock: ManualClock = clock self._connection_seconds: float = connection_seconds self._compile_seconds: float = compile_seconds - self._dialect_seconds: list[float] = [0.0, rules_seconds] + self._dialect_seconds: list[float] = [0.0, 0.0, rules_seconds] def expression_inference_profile(self) -> ExpressionInferenceProfile: self._clock.advance(self._compile_seconds) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py index a6f0961b84..27f64d2635 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py @@ -19,3 +19,4 @@ class RuntimeRelationGuardTestCase: owner_label: str = "task 'export_orders'" owner_kind: HardCodedRelationOwnerKind = HardCodedRelationOwnerKind.NODE upstream_loader_by_source: dict[str, str] = field(default_factory=dict) + dialect: str = "duckdb" diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py index d178f0f5bd..54a52f618f 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py @@ -108,6 +108,29 @@ expected_warning_fragments=(), expected_warning_count=0, ), + RuntimeRelationGuardTestCase( + description="ClickHouse TRUNCATE, ALTER DELETE and INTO OUTFILE warn", + statements=( + "TRUNCATE TABLE main.orders", + "ALTER TABLE main.orders DELETE WHERE order_id = 1", + "SELECT * FROM main.customers INTO OUTFILE 'customers.csv'", + ), + expected_warning_fragments=("named model:orders", "named model:customers"), + expected_warning_count=2, + dialect="clickhouse", + ), + RuntimeRelationGuardTestCase( + description="ClickHouse temporary table, OPTIMIZE and ALTER UPDATE do not warn", + statements=( + "CREATE TEMPORARY TABLE customers AS SELECT 1 AS id", + "SELECT * FROM customers", + "OPTIMIZE TABLE main.orders FINAL", + "ALTER TABLE main.orders UPDATE status = 'x' WHERE 1", + ), + expected_warning_fragments=(), + expected_warning_count=0, + dialect="clickhouse", + ), RuntimeRelationGuardTestCase( description="unparseable SQL is not blocked and does not warn", statements=("VACUUM ((( customers",), @@ -125,7 +148,7 @@ def test_given_python_sql_when_checking_then_warns_for_hard_coded_project_relati owner_label=test_case.owner_label, owner_kind=test_case.owner_kind, project_relations=_PROJECT_RELATIONS, - dialect="duckdb", + dialect=test_case.dialect, default_database=None, default_schema="main", warnings=warnings,