From 8438a9a29d8944d181c7cae0866bed17d9b2672d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:13:43 +0000 Subject: [PATCH] Choose the planner proofs' parser through the analysis route Migration query shapes and fingerprints, raw diff query shapes, relation names, fixture star keys and Snowflake's metadata-only CREATE now parse a native dialect's SQL with SQLQuack, and keep Polyglot only on the legacy route. SQL the native parser cannot read whole gets no proof, so the planner takes its conservative path. Cursor types parse unknown warehouse types with SQLQuack's parser. NativeDialect gains parse, and read_whole becomes a provided method over it, as the analysis route's follow-up defines them. --- .../_helpers/clickhouse_dialect.rs | 30 +---- .../src/analysis_route/constants.rs | 2 + .../src/analysis_route/tests.rs | 2 + .../tests/test_planner_routes.rs | 103 ++++++++++++++++++ .../src/analysis_route/tests/test_types.rs | 9 +- .../src/analysis_route/types.rs | 29 ++++- .../diff_selection/main/raw_query_shape.rs | 44 +++++++- .../src/planner_fixtures/main/star_keys.rs | 71 ++++++++++++ .../_helpers/proofs/fingerprint.rs | 8 +- .../_helpers/proofs/parse.rs | 54 ++++++++- .../src/planner_migrations/constants.rs | 4 + .../planner_strategy/_helpers/type_names.rs | 33 +++++- .../src/planner_strategy/constants.rs | 8 +- .../src/planner_strategy/errors.rs | 2 +- .../main/classify_cursor_sql_type.rs | 18 ++- .../main/cursor_type_warning.rs | 2 +- .../src/relation_names/_helpers/walk.rs | 39 ++++++- .../src/snowflake_cost/_helpers/statements.rs | 29 ++++- .../src/snowflake_cost/constants.rs | 2 + .../_helpers/planner_strategy/cursor_types.rs | 2 +- 20 files changed, 433 insertions(+), 58 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs index 4b68b5b914..5926fd632d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs @@ -7,9 +7,7 @@ use sqlquack_syntax::Parsed; use sqlquack_syntax_clickhouse::parse_strict; use crate::analysis_route::_helpers::clickhouse_spelling::clickhouse_spelling; -use crate::analysis_route::constants::{ - CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX, STATEMENTS_CONSTRUCT, -}; +use crate::analysis_route::constants::{CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX}; use crate::analysis_route::main::clickhouse_type_match::clickhouse_type_match; use crate::analysis_route::models::{ClickHouseDialect, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; @@ -20,19 +18,11 @@ impl NativeDialect for ClickHouseDialect { CLICKHOUSE_DIALECT } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { - let parsed: Parsed = self.parse(sql)?; - match (parsed.gaps.first(), parsed.statements.len()) { - (Some(gap), _) => Err(ParserGap { - construct: gap.construct.to_owned(), - offset: gap.start, - }), - (None, 1) => Ok(()), - (None, _) => Err(ParserGap { - construct: STATEMENTS_CONSTRUCT.to_owned(), - offset: sql.len(), - }), - } + fn parse(&self, sql: &str) -> Result { + parse_strict(sql).map_err(|error| ParserGap { + construct: error.message, + offset: error.position, + }) } fn stub_prefix(&self) -> &'static str { @@ -41,14 +31,6 @@ impl NativeDialect for ClickHouseDialect { } impl ClickHouseDialect { - /// Every statement of `sql` with the parser's named gaps, or the first error. - pub fn parse(&self, sql: &str) -> Result { - parse_strict(sql).map_err(|error| ParserGap { - construct: error.message, - offset: error.position, - }) - } - /// The ClickHouse profile: names compared with case, types as ClickHouse 26.1 reports them. #[must_use] pub fn profile(&self) -> &'static dyn DialectProfile { diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 13829559bf..c7caee4f36 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -19,6 +19,8 @@ pub(crate) const CLICKHOUSE_DIALECT: &str = "clickhouse"; pub(crate) const CLICKHOUSE_STUB_PREFIX: &str = "__sqlbuild_"; /// The construct of SQL holding more than one statement. pub(crate) const STATEMENTS_CONSTRUCT: &str = "more than one statement"; +/// The construct of a statement a parser kept as unparsed text. +pub(crate) const UNPARSED_STATEMENT_CONSTRUCT: &str = "unparsed statement"; /// Each SQL type alias whose name Polyglot respells for ClickHouse, case-insensitive, with the /// spelling SQLBuild's type fallback reads as Polyglot's canonical name (`{}` keeps the arguments) /// and, where the fallback's family differs from Polyglot's, Polyglot's family at the top level. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index f65158f54e..b7f155e5f0 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -4,5 +4,7 @@ mod test_clickhouse_dialect; mod test_clickhouse_type_match; #[path = "tests/test_native_gap.rs"] mod test_native_gap; +#[path = "tests/test_planner_routes.rs"] +mod test_planner_routes; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs new file mode 100644 index 0000000000..26cb06be45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs @@ -0,0 +1,103 @@ +use std::collections::BTreeMap; + +use crate::diff_selection::main::raw_query_shape::raw_query_shape; +use crate::planner_fixtures::main::star_keys::star_keys; +use crate::planner_fixtures::models::{FixtureColumn, FixtureStarRelation}; +use crate::planner_migrations::main::migration_fingerprint::migration_fingerprint; +use crate::planner_migrations::main::query_shape::query_shape; +use crate::relation_names::main::extract_relation_names::extract_relation_names; +use crate::relation_names::models::RelationName; + +/// A ClickHouse query the native parser reads, and one it stops at (a named gap). +const READABLE: &str = "SELECT o.id, o.amount AS total FROM orders AS o"; +const UNREADABLE: &str = "SELECT o.id FROM orders AS o INTO OUTFILE 'x.csv'"; + +#[test] +fn given_clickhouse_sql_when_proving_a_column_migration_then_only_sql_read_whole_has_a_shape() { + let shape = query_shape(READABLE, "clickhouse") + .expect("no internal failure") + .expect("readable SQL has a shape"); + let names: Vec<&str> = shape.projections.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["id", "total"]); + assert!( + query_shape(UNREADABLE, "clickhouse") + .expect("no internal failure") + .is_none() + ); +} + +#[test] +fn given_clickhouse_sql_when_fingerprinting_a_migration_then_formatting_is_ignored_and_gaps_have_none() + { + let identities: BTreeMap = BTreeMap::new(); + let fingerprint = |sql: &str| migration_fingerprint(sql, "", &identities, "clickhouse"); + let base = fingerprint(READABLE).expect("no internal failure"); + assert!(base.is_some()); + assert_eq!( + fingerprint("SELECT o.id,\n o.amount AS total -- note\nFROM orders AS o").expect("ok"), + base + ); + assert_ne!( + fingerprint("SELECT o.id, o.amount AS sum FROM orders AS o").expect("ok"), + base + ); + assert_eq!(fingerprint(UNREADABLE).expect("ok"), None); +} + +#[test] +fn given_a_clickhouse_diff_query_when_reading_its_shape_then_unreadable_sql_is_refused() { + let shape = raw_query_shape(READABLE, "clickhouse").expect("readable"); + assert_eq!(shape.root_key, "select"); + assert!(!shape.writes_or_locks); + assert!(raw_query_shape(UNREADABLE, "clickhouse").is_err()); +} + +#[test] +fn given_clickhouse_sql_when_extracting_relation_names_then_ctes_are_skipped_and_gaps_have_none() { + let extracted = extract_relation_names( + "WITH c AS (SELECT 1 AS id) SELECT * FROM shop.orders AS o JOIN c ON o.id = c.id", + Some("clickhouse"), + ) + .expect("no failure") + .expect("readable"); + assert_eq!( + extracted.relations, + [RelationName { + name: "orders".to_owned(), + schema: Some("shop".to_owned()), + database: None, + }] + ); + assert_eq!( + extract_relation_names(UNREADABLE, Some("clickhouse")).expect("no failure"), + None + ); +} + +#[test] +fn given_a_clickhouse_star_when_selecting_fixture_dependencies_then_unsettled_stars_take_every_relation() + { + let relation = |name: &str| FixtureStarRelation { + key: ("model".to_owned(), name.to_owned()), + columns: vec![FixtureColumn { + name: "id".to_owned(), + folded_name: "id".to_owned(), + sql_type: Some("Int32".to_owned()), + nullable: Some(false), + }], + authoritative: true, + }; + let relations = [relation("orders"), relation("customers")]; + let keys = |sql: &str| star_keys(sql, &relations, Some("clickhouse"), None); + let orders = vec![("model".to_owned(), "orders".to_owned())]; + let every: Vec<(String, String)> = relations.iter().map(|r| r.key.clone()).collect(); + let joined = "FROM orders AS o JOIN customers AS c ON o.id = c.id"; + assert_eq!(keys(&format!("SELECT o.* {joined}")), orders); + assert_eq!(keys(&format!("SELECT count(*) AS n {joined}")), Vec::new()); + assert_eq!(keys(&format!("SELECT * {joined}")), every); + assert_eq!(keys(&format!("SELECT zz.* {joined}")), every); + assert_eq!( + keys(&format!("SELECT o.* {joined} INTO OUTFILE 'x.csv'")), + every + ); +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9587ebda22..3671f2a487 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,7 @@ use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; +use sqlquack_syntax::expressions::names::models::Null; +use sqlquack_syntax::{Expression, Parsed}; pub(super) struct NativeGapTestCase { pub(super) description: &'static str, @@ -31,7 +33,7 @@ impl NativeDialect for PivotlessDialect { "postgres" } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { + fn parse(&self, sql: &str) -> Result { let refused = ["PIVOT", "@@@", "__sqlbuild_udf_", "__cursor_", "@'"] .iter() .filter_map(|word| sql.find(word).map(|offset| (offset, *word))) @@ -41,7 +43,10 @@ impl NativeDialect for PivotlessDialect { construct: word.to_owned(), offset, }), - None => Ok(()), + None => Ok(Parsed { + statements: vec![Expression::Null(Null)], + gaps: Vec::new(), + }), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index fedc80ad24..9accb9c990 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,5 +1,8 @@ //! What a dialect's native analysis module provides. +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::constants::{STATEMENTS_CONSTRUCT, UNPARSED_STATEMENT_CONSTRUCT}; use crate::analysis_route::models::ParserGap; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. @@ -7,8 +10,30 @@ pub trait NativeDialect: Sync { /// The adapter's analysis dialect name, such as `postgres`. fn name(&self) -> &'static str; - /// Whether the native parser reads `sql` whole: one statement, no gap and no unparsed bytes. - fn read_whole(&self, sql: &str) -> Result<(), ParserGap>; + /// Every statement of `sql` with the parser's named gaps, or the first error. + fn parse(&self, sql: &str) -> Result; + + /// The one statement of `sql`, with no gap and not a lone unparsed command. + fn read_whole(&self, sql: &str) -> Result { + let mut parsed: Parsed = self.parse(sql)?; + if let Some(gap) = parsed.gaps.first() { + return Err(ParserGap { + construct: gap.construct.to_owned(), + offset: gap.start, + }); + } + let unread = |construct: &str| ParserGap { + construct: construct.to_owned(), + offset: sql.len(), + }; + if parsed.statements.len() != 1 { + return Err(unread(STATEMENTS_CONSTRUCT)); + } + match parsed.statements.remove(0) { + Expression::Command(_) => Err(unread(UNPARSED_STATEMENT_CONSTRUCT)), + statement => Ok(statement), + } + } /// The engine-accepted prefix of UDF and cursor stubs: `__sqlbuild_`, or `__sqlbuild.` for T-SQL. fn stub_prefix(&self) -> &'static str; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs index c553e213e1..112d2aff10 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -1,14 +1,54 @@ -//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. +//! Parse one raw diff query with the dialect's parser and read its shape: SQLQuack where native, +//! else as the Polyglot wheel's guarded `parse_one` does. use polyglot_sql::dialects::Dialect; use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; +use sqlquack_syntax::traversal::types::ExpressionWalk as NativeWalk; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; use crate::diff_selection::models::RawQueryShape; -/// The query's shape, or the text of the exception the wheel raises for it. +/// The query's shape, or why it is not one statement the dialect's parser reads whole. pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { + match analysis_route(Some(dialect)) { + AnalysisRoute::Native(native) => { + let root: sqlquack_syntax::Expression = native + .read_whole(sql) + .map_err(|gap| format!("the {dialect} parser cannot read {}", gap.construct))?; + native_shape(&root) + } + AnalysisRoute::Legacy => polyglot_shape(sql, dialect), + } +} + +/// The shape of a statement SQLQuack read whole. +fn native_shape(root: &sqlquack_syntax::Expression) -> Result { + let root_key: String = root.variant_name().to_lowercase(); + let node_keys: Vec = if RAW_QUERY_READ_ONLY_ROOTS.contains(&root_key.as_str()) { + NativeWalk::dfs(root) + .map(|node| node.variant_name().to_lowercase()) + .collect() + } else { + Vec::new() + }; + let writes_or_locks: bool = match root { + sqlquack_syntax::Expression::Select(select) => { + select.into.is_some() || !select.locks.is_empty() + } + _ => false, + }; + Ok(RawQueryShape { + root_key, + node_keys, + writes_or_locks, + }) +} + +/// The shape Polyglot reads, or the text of the exception the wheel raises. +fn polyglot_shape(sql: &str, dialect: &str) -> Result { let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; let guard: ComplexityGuardOptions = diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs index 5afda5f19d..c3848a541a 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -7,7 +7,12 @@ use polyglot_sql::{ AnalyzeQueryOptions, DialectType, SchemaColumn, SchemaTable, ValidationSchema, analyze_query, }; use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::Expression as NativeExpression; +use sqlquack_syntax::traversal::types::ExpressionWalk as NativeWalk; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; use crate::planner_fixtures::models::FixtureStarRelation; @@ -32,6 +37,9 @@ pub fn star_keys( { return Vec::new(); } + if let AnalysisRoute::Native(native) = analysis_route(dialect) { + return native_star_keys(native, sql, relations); + } let Ok(dialect) = DialectType::from_str(dialect.filter(|name| !name.is_empty()).unwrap_or("generic")) else { @@ -123,3 +131,66 @@ fn column_type(sql_type: Option<&str>) -> String { .unwrap_or("UNKNOWN") .to_owned() } + +/// Star dependencies from SQLQuack's tree, a superset of what each star reads: a bare star selects +/// every relation in the query, and a star whose qualifier the tree does not settle, or SQL the +/// parser cannot read whole, selects every relation given. +fn native_star_keys( + native: &dyn NativeDialect, + sql: &str, + relations: &[FixtureStarRelation], +) -> Vec<(String, String)> { + let every = || { + relations + .iter() + .map(|relation| relation.key.clone()) + .collect() + }; + let Ok(sql) = stub_markers(sql) else { + return every(); + }; + let Ok(root) = native.read_whole(&sql) else { + return every(); + }; + let mut aliases: HashMap = HashMap::new(); + let mut stars: Vec> = Vec::new(); + for node in NativeWalk::dfs(&root) { + match node { + NativeExpression::Table(table) => { + let name: String = python_casefold(&table.name.name); + aliases.insert(name.clone(), name.clone()); + if let Some(alias) = &table.alias { + aliases.insert(python_casefold(&alias.name), name); + } + } + NativeExpression::Select(select) => { + stars.extend(select.expressions.iter().filter_map(|projection| { + match projection { + NativeExpression::Star(star) => Some( + star.table + .as_ref() + .map(|table| python_casefold(&table.name)), + ), + _ => None, + } + })); + } + _ => {} + } + } + let mut names: HashSet = HashSet::new(); + for star in &stars { + match star.as_ref().map(|qualifier| aliases.get(qualifier)) { + Some(Some(name)) => { + names.insert(name.clone()); + } + None => names.extend(aliases.values().cloned()), + Some(None) => return every(), + } + } + relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs index 338624014a..1df5d5f24a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs @@ -1,11 +1,14 @@ //! Normalize migration metadata and SQL without changing the persisted format. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::planner_changes::main::version_identity_payload::version_identity_payload; use crate::planner_changes::models::IdentityPayload; use crate::planner_migrations::_helpers::proofs::ast::strip_formatting; +use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::_helpers::proofs::local_names::{local_names, rewrite, unambiguous}; use crate::planner_migrations::_helpers::proofs::parse::parse; -use crate::planner_migrations::constants::{EXCLUDED_CONFIG, REF_MARKER}; +use crate::planner_migrations::constants::{EXCLUDED_CONFIG, NATIVE_TREE_PREFIX, REF_MARKER}; use polyglot_sql::{Expression, dialects::Dialect}; use regex::Regex; use serde_json::Value; @@ -74,6 +77,9 @@ pub(crate) fn normalized( } } rewrite(&mut parsed, &names, &ctes); + if let AnalysisRoute::Native(_) = analysis_route(Some(dialect)) { + return encode(&parsed).map(|tree| Some(format!("{NATIVE_TREE_PREFIX}{tree}"))); + } let expression: Expression = serde_json::from_value(parsed).map_err(|error| error.to_string())?; Dialect::get_by_name(dialect) diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs index a84b3ee534..f21c678c3a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs @@ -1,11 +1,46 @@ -//! Parse migration SQL under the compiler's complexity limit. +//! Parse migration SQL with the dialect's parser: SQLQuack where native, else Polyglot under the +//! compiler's complexity limit. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::planner_migrations::constants::{SET_OPERATION_LEFT, WITH_KEY}; use polyglot_sql::dialects::Dialect; use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; use serde_json::{Value, json}; +/// The one statement of `sql` as a tree, or None when there is no proof: the parser cannot read it +/// whole, which callers take as the conservative plan. pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { + let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) else { + return polyglot_parse(sql, dialect); + }; + let Ok(statement) = native.read_whole(sql) else { + return Ok(None); + }; + let mut tree: Value = serde_json::to_value(statement).map_err(|error| error.to_string())?; + hoist_leading_with(&mut tree); + Ok(Some(tree)) +} + +/// Move a set operation's leading WITH from its first SELECT, where ClickHouse's parser hangs it, +/// onto the set operation, where the proofs read a statement's CTEs. +fn hoist_leading_with(tree: &mut Value) { + let Some(operation) = tree + .as_object_mut() + .and_then(|object| object.values_mut().next()) + .filter(|operation| operation.get(SET_OPERATION_LEFT).is_some()) + else { + return; + }; + if !operation[WITH_KEY].is_null() { + return; + } + let with: Value = leading_with(&mut operation[SET_OPERATION_LEFT]); + operation[WITH_KEY] = with; +} + +fn polyglot_parse(sql: &str, dialect: &str) -> Result, String> { let dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("unknown dialect {dialect}"))?; let guard: ComplexityGuardOptions = @@ -24,3 +59,20 @@ pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { .map(Some) .map_err(|error| error.to_string()) } + +/// The WITH of the first SELECT under `node`, taken out of it; null when it has none. +fn leading_with(node: &mut Value) -> Value { + let Some(inner) = node + .as_object_mut() + .and_then(|object| object.values_mut().next()) + else { + return Value::Null; + }; + if inner.get(SET_OPERATION_LEFT).is_some() { + return leading_with(&mut inner[SET_OPERATION_LEFT]); + } + inner + .as_object_mut() + .and_then(|select| select.insert(WITH_KEY.to_owned(), Value::Null)) + .unwrap_or(Value::Null) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs index b760634f78..1cd92ada6a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -24,6 +24,8 @@ pub(crate) const OPAQUE_TUPLE_FIELDS: &[&str] = &[ ]; pub(crate) const HAVING_MAX_KEY: &str = "having_max"; pub(crate) const WITH_KEY: &str = "with"; +/// A set operation's first branch. +pub(crate) const SET_OPERATION_LEFT: &str = "left"; pub(crate) const TABLE_KEY: &str = "table"; pub(crate) const NUMBER_LITERAL: &str = "number"; @@ -49,3 +51,5 @@ pub(crate) const POLICY_FAIL: &str = "fail"; pub(crate) const MODEL_DONE: &str = "done"; pub(crate) const MODEL_RENAMED: &str = "renamed"; pub(crate) const MODEL_CONFLICT: &str = "conflict"; +/// Marks a native dialect's normalized SQL, its canonical tree, so it never equals Polyglot's SQL text. +pub(crate) const NATIVE_TREE_PREFIX: &str = "sqlquack-tree:"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs index aae5b546d0..9294a01917 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs @@ -1,8 +1,12 @@ -//! Cursor types of warehouse type names and Polyglot type arguments. +//! Cursor types of warehouse type names and parsed type arguments. use serde_json::{Map, Value}; +use sqlquack_syntax::traversal::types::ExpressionWalk; +use sqlquack_syntax::{Expression, ParseOptions}; -use crate::planner_strategy::constants::{INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES}; +use crate::planner_strategy::constants::{ + INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES, TYPE_PARSE_DIALECT, +}; use crate::planner_strategy::models::CursorType; /// The cursor type of a known type name. @@ -25,3 +29,28 @@ pub(crate) fn argument(args: &Map, key: &str) -> String { Some(other) => other.to_string().to_uppercase(), } } + +/// The scalar arguments of `type_sql` as SQLQuack's parser reads it in a cast, or why not. +pub(crate) fn type_args(type_sql: &str) -> Result, String> { + let statements: Vec = sqlquack_syntax::parse_strict( + &format!("SELECT CAST(NULL AS {type_sql})"), + TYPE_PARSE_DIALECT, + &ParseOptions::default(), + ) + .map_err(|error| error.to_string())?; + let data_type = statements + .iter() + .flat_map(|statement| statement.dfs()) + .find_map(|node| match node { + Expression::Cast(cast) => Some(&cast.to), + _ => None, + }) + .ok_or_else(|| format!("not one type: {type_sql}"))?; + match serde_json::to_value(data_type).map_err(|error| error.to_string())? { + Value::Object(fields) => Ok(fields + .into_iter() + .filter(|(_, value)| !matches!(value, Value::Object(_) | Value::Array(_))) + .collect()), + other => Err(format!("not one type: {other}")), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 1176dc46a1..ad15667c5d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -72,10 +72,10 @@ pub const INTEGER_TYPE_NAMES: [&str; 17] = [ ]; /// Decimal type names whose zero-scale form is an integer cursor. pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; -/// The type Polyglot reports for a type it does not know. -pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; -/// The dialect cursor type classification parses unknown types with. -pub const GENERIC_DIALECT: &str = "generic"; +/// The type the parser reports for a type it does not know. +pub const CUSTOM_TYPE_NAME: &str = "CUSTOM"; +/// The parser cursor type classification reads an unknown warehouse type with. +pub(crate) const TYPE_PARSE_DIALECT: sqlquack_syntax::Dialect = sqlquack_syntax::Dialect::DuckDB; /// Error code of an authored cursor input column its input does not expose. pub const CURSOR_INPUT_COLUMN_ERROR_CODE: &str = "S302"; /// Incremental strategies whose microbatches replay one batch by default. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs index 2be7a8dd5b..f0b4a53575 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs @@ -84,7 +84,7 @@ impl CappedWatermarkInput { } } -/// A warehouse type Polyglot could not parse, which classifies as no cursor type. +/// A warehouse type the parser could not read, which classifies as no cursor type. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TypeParseFailure { pub normalized_type: String, diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs index 22dedf16bd..4853f55f17 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs @@ -3,13 +3,12 @@ use serde_json::{Map, Value}; use crate::planner_strategy::_helpers::decimal_scale::decimal_scale_is_zero; -use crate::planner_strategy::_helpers::type_names::{argument, named_type}; -use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME}; +use crate::planner_strategy::_helpers::type_names::{argument, named_type, type_args}; +use crate::planner_strategy::constants::CUSTOM_TYPE_NAME; use crate::planner_strategy::errors::TypeParseFailure; use crate::planner_strategy::models::CursorType; -use crate::type_system::main::polyglot_type_args::polyglot_type_args; -/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type. +/// Timestamp or integer for supported cross-adapter types; `Err` when the parser cannot read an unknown type. pub fn classify_cursor_sql_type(sql_type: &str) -> Result, TypeParseFailure> { let normalized: String = sql_type .to_uppercase() @@ -28,13 +27,12 @@ pub fn classify_cursor_sql_type(sql_type: &str) -> Result, Ty if let Some(cursor_type) = named_type(&base) { return Ok(Some(cursor_type)); } - let args: Map = - polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure { - normalized_type: normalized.clone(), - message, - })?; + let args: Map = type_args(&normalized).map_err(|message| TypeParseFailure { + normalized_type: normalized.clone(), + message, + })?; let mut type_name: String = argument(&args, "data_type"); - if type_name == POLYGLOT_CUSTOM_TYPE_NAME { + if type_name == CUSTOM_TYPE_NAME { type_name = argument(&args, "name").replace('_', ""); } Ok(named_type(&type_name)) diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs index d7b05b6719..9293db1051 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs @@ -11,7 +11,7 @@ fn cursor_type_name(cursor_type: CursorType) -> &'static str { } } -/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any Polyglot parse error that left it unclassified. +/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any parse error that left it unclassified. pub fn cursor_type_warning( check: &CursorTypeCheck, ) -> (Option, Option) { diff --git a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs index 28fe14df79..277325f0fb 100644 --- a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs @@ -4,6 +4,8 @@ use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; use serde_json::{Map, Value, json}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::relation_names::constants::{ CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, NAME_KEY, QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, @@ -19,14 +21,12 @@ pub(crate) fn extracted_relations( let dialect: &str = dialect .filter(|name| !name.is_empty()) .unwrap_or(GENERIC_DIALECT); - let Some(statements) = parsed_statements(sql, dialect)? else { + let Some(trees) = statement_trees(sql, dialect)? else { return Ok(None); }; let mut extracted: ExtractedRelations = ExtractedRelations::default(); - for statement in &statements { - let tree: Value = serde_json::to_value(statement) - .map_err(|error| RelationNamesFailure::Internal(error.to_string()))?; - let nodes: Vec<&Map> = walked(&tree); + for tree in &trees { + let nodes: Vec<&Map> = walked(tree); let cte_names: Vec = nodes.iter().flat_map(|node| cte_names(node)).collect(); extracted .temporary @@ -45,6 +45,35 @@ pub(crate) fn extracted_relations( Ok(Some(extracted)) } +/// Each statement as a tree: SQLQuack's where native, with any gap or error None; else Polyglot's. +fn statement_trees(sql: &str, dialect: &str) -> Result>, RelationNamesFailure> { + let internal = |error: serde_json::Error| RelationNamesFailure::Internal(error.to_string()); + if let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) { + if sql.trim().is_empty() { + return Ok(Some(Vec::new())); + } + return match native.parse(sql) { + Ok(parsed) if parsed.gaps.is_empty() => parsed + .statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal), + _ => Ok(None), + }; + } + let Some(statements) = parsed_statements(sql, dialect)? else { + return Ok(None); + }; + statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal) +} + /// The wheel's guarded `parse`: Ok(None) is the `PolyglotError` Python catches. fn parsed_statements( sql: &str, diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs index 7e49d9c53d..a9dea2d44e 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs @@ -2,7 +2,11 @@ use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; -use crate::snowflake_cost::constants::{CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER}; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::snowflake_cost::constants::{ + CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER, SNOWFLAKE_DIALECT, +}; pub(crate) fn of_kind(query_type: &str, kind: &str) -> bool { query_type == kind @@ -16,8 +20,14 @@ pub(crate) fn python_space(character: char) -> bool { character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) } -/// Whether a CREATE statement's text parses as a definition that needs no warehouse. +/// Whether a CREATE statement's text parses as a definition that needs no warehouse; false, the +/// warehouse-billed reading, when the parser cannot read it whole. pub(crate) fn metadata_only_create(query_text: &str) -> bool { + if let AnalysisRoute::Native(native) = analysis_route(Some(SNOWFLAKE_DIALECT)) { + return native + .read_whole(query_text) + .is_ok_and(|statement| native_metadata_only(&statement)); + } let options: ParseOptions = ParseOptions { complexity_guard: None, }; @@ -41,3 +51,18 @@ pub(crate) fn metadata_only_create(query_text: &str) -> bool { statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), } } + +/// Whether a statement SQLQuack read is a CREATE that needs no warehouse. +fn native_metadata_only(statement: &sqlquack_syntax::Expression) -> bool { + match statement { + sqlquack_syntax::Expression::CreateTable(table) => { + !(table.as_select.is_some() + || table.clone_source.is_some() + || table.using_template.is_some() + || table.table_modifier.as_deref().unwrap_or("").to_uppercase() + == DYNAMIC_TABLE_MODIFIER) + } + sqlquack_syntax::Expression::CreateView(view) if view.materialized => false, + statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs index 67a32cb1f1..9eae6ee8d0 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -8,3 +8,5 @@ pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; pub const QUERY_TAG_APP: &str = "sqlbuild"; pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; +/// Snowflake's analysis dialect name, which chooses its parser. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index 48f55a3095..840b93dcea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::records::ids::RecordId; const PLANNER_LOGGER: &str = "sqlbuild.planner"; -/// Log the Polyglot parse error that left a warehouse type unclassified. +/// Log the parse error that left a warehouse type unclassified. fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { let text = |value: &str| PyString::new(py, value).into_any(); call_with(