diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs index 4b68b5b914..5926fd632d 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_dialect.rs @@ -7,9 +7,7 @@ use sqlquack_syntax::Parsed; use sqlquack_syntax_clickhouse::parse_strict; use crate::analysis_route::_helpers::clickhouse_spelling::clickhouse_spelling; -use crate::analysis_route::constants::{ - CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX, STATEMENTS_CONSTRUCT, -}; +use crate::analysis_route::constants::{CLICKHOUSE_DIALECT, CLICKHOUSE_STUB_PREFIX}; use crate::analysis_route::main::clickhouse_type_match::clickhouse_type_match; use crate::analysis_route::models::{ClickHouseDialect, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; @@ -20,19 +18,11 @@ impl NativeDialect for ClickHouseDialect { CLICKHOUSE_DIALECT } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { - let parsed: Parsed = self.parse(sql)?; - match (parsed.gaps.first(), parsed.statements.len()) { - (Some(gap), _) => Err(ParserGap { - construct: gap.construct.to_owned(), - offset: gap.start, - }), - (None, 1) => Ok(()), - (None, _) => Err(ParserGap { - construct: STATEMENTS_CONSTRUCT.to_owned(), - offset: sql.len(), - }), - } + fn parse(&self, sql: &str) -> Result { + parse_strict(sql).map_err(|error| ParserGap { + construct: error.message, + offset: error.position, + }) } fn stub_prefix(&self) -> &'static str { @@ -41,14 +31,6 @@ impl NativeDialect for ClickHouseDialect { } impl ClickHouseDialect { - /// Every statement of `sql` with the parser's named gaps, or the first error. - pub fn parse(&self, sql: &str) -> Result { - parse_strict(sql).map_err(|error| ParserGap { - construct: error.message, - offset: error.position, - }) - } - /// The ClickHouse profile: names compared with case, types as ClickHouse 26.1 reports them. #[must_use] pub fn profile(&self) -> &'static dyn DialectProfile { diff --git a/crates/sqlbuild-analysis/src/analysis_route/constants.rs b/crates/sqlbuild-analysis/src/analysis_route/constants.rs index 13829559bf..c7caee4f36 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/constants.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/constants.rs @@ -19,6 +19,8 @@ pub(crate) const CLICKHOUSE_DIALECT: &str = "clickhouse"; pub(crate) const CLICKHOUSE_STUB_PREFIX: &str = "__sqlbuild_"; /// The construct of SQL holding more than one statement. pub(crate) const STATEMENTS_CONSTRUCT: &str = "more than one statement"; +/// The construct of a statement a parser kept as unparsed text. +pub(crate) const UNPARSED_STATEMENT_CONSTRUCT: &str = "unparsed statement"; /// Each SQL type alias whose name Polyglot respells for ClickHouse, case-insensitive, with the /// spelling SQLBuild's type fallback reads as Polyglot's canonical name (`{}` keeps the arguments) /// and, where the fallback's family differs from Polyglot's, Polyglot's family at the top level. diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index f65158f54e..b7f155e5f0 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -4,5 +4,7 @@ mod test_clickhouse_dialect; mod test_clickhouse_type_match; #[path = "tests/test_native_gap.rs"] mod test_native_gap; +#[path = "tests/test_planner_routes.rs"] +mod test_planner_routes; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs new file mode 100644 index 0000000000..26cb06be45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_planner_routes.rs @@ -0,0 +1,103 @@ +use std::collections::BTreeMap; + +use crate::diff_selection::main::raw_query_shape::raw_query_shape; +use crate::planner_fixtures::main::star_keys::star_keys; +use crate::planner_fixtures::models::{FixtureColumn, FixtureStarRelation}; +use crate::planner_migrations::main::migration_fingerprint::migration_fingerprint; +use crate::planner_migrations::main::query_shape::query_shape; +use crate::relation_names::main::extract_relation_names::extract_relation_names; +use crate::relation_names::models::RelationName; + +/// A ClickHouse query the native parser reads, and one it stops at (a named gap). +const READABLE: &str = "SELECT o.id, o.amount AS total FROM orders AS o"; +const UNREADABLE: &str = "SELECT o.id FROM orders AS o INTO OUTFILE 'x.csv'"; + +#[test] +fn given_clickhouse_sql_when_proving_a_column_migration_then_only_sql_read_whole_has_a_shape() { + let shape = query_shape(READABLE, "clickhouse") + .expect("no internal failure") + .expect("readable SQL has a shape"); + let names: Vec<&str> = shape.projections.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["id", "total"]); + assert!( + query_shape(UNREADABLE, "clickhouse") + .expect("no internal failure") + .is_none() + ); +} + +#[test] +fn given_clickhouse_sql_when_fingerprinting_a_migration_then_formatting_is_ignored_and_gaps_have_none() + { + let identities: BTreeMap = BTreeMap::new(); + let fingerprint = |sql: &str| migration_fingerprint(sql, "", &identities, "clickhouse"); + let base = fingerprint(READABLE).expect("no internal failure"); + assert!(base.is_some()); + assert_eq!( + fingerprint("SELECT o.id,\n o.amount AS total -- note\nFROM orders AS o").expect("ok"), + base + ); + assert_ne!( + fingerprint("SELECT o.id, o.amount AS sum FROM orders AS o").expect("ok"), + base + ); + assert_eq!(fingerprint(UNREADABLE).expect("ok"), None); +} + +#[test] +fn given_a_clickhouse_diff_query_when_reading_its_shape_then_unreadable_sql_is_refused() { + let shape = raw_query_shape(READABLE, "clickhouse").expect("readable"); + assert_eq!(shape.root_key, "select"); + assert!(!shape.writes_or_locks); + assert!(raw_query_shape(UNREADABLE, "clickhouse").is_err()); +} + +#[test] +fn given_clickhouse_sql_when_extracting_relation_names_then_ctes_are_skipped_and_gaps_have_none() { + let extracted = extract_relation_names( + "WITH c AS (SELECT 1 AS id) SELECT * FROM shop.orders AS o JOIN c ON o.id = c.id", + Some("clickhouse"), + ) + .expect("no failure") + .expect("readable"); + assert_eq!( + extracted.relations, + [RelationName { + name: "orders".to_owned(), + schema: Some("shop".to_owned()), + database: None, + }] + ); + assert_eq!( + extract_relation_names(UNREADABLE, Some("clickhouse")).expect("no failure"), + None + ); +} + +#[test] +fn given_a_clickhouse_star_when_selecting_fixture_dependencies_then_unsettled_stars_take_every_relation() + { + let relation = |name: &str| FixtureStarRelation { + key: ("model".to_owned(), name.to_owned()), + columns: vec![FixtureColumn { + name: "id".to_owned(), + folded_name: "id".to_owned(), + sql_type: Some("Int32".to_owned()), + nullable: Some(false), + }], + authoritative: true, + }; + let relations = [relation("orders"), relation("customers")]; + let keys = |sql: &str| star_keys(sql, &relations, Some("clickhouse"), None); + let orders = vec![("model".to_owned(), "orders".to_owned())]; + let every: Vec<(String, String)> = relations.iter().map(|r| r.key.clone()).collect(); + let joined = "FROM orders AS o JOIN customers AS c ON o.id = c.id"; + assert_eq!(keys(&format!("SELECT o.* {joined}")), orders); + assert_eq!(keys(&format!("SELECT count(*) AS n {joined}")), Vec::new()); + assert_eq!(keys(&format!("SELECT * {joined}")), every); + assert_eq!(keys(&format!("SELECT zz.* {joined}")), every); + assert_eq!( + keys(&format!("SELECT o.* {joined} INTO OUTFILE 'x.csv'")), + every + ); +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index 9587ebda22..3671f2a487 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -1,5 +1,7 @@ use crate::analysis_route::models::{LocatedGap, ParserGap, TypeMatch}; use crate::analysis_route::types::NativeDialect; +use sqlquack_syntax::expressions::names::models::Null; +use sqlquack_syntax::{Expression, Parsed}; pub(super) struct NativeGapTestCase { pub(super) description: &'static str, @@ -31,7 +33,7 @@ impl NativeDialect for PivotlessDialect { "postgres" } - fn read_whole(&self, sql: &str) -> Result<(), ParserGap> { + fn parse(&self, sql: &str) -> Result { let refused = ["PIVOT", "@@@", "__sqlbuild_udf_", "__cursor_", "@'"] .iter() .filter_map(|word| sql.find(word).map(|offset| (offset, *word))) @@ -41,7 +43,10 @@ impl NativeDialect for PivotlessDialect { construct: word.to_owned(), offset, }), - None => Ok(()), + None => Ok(Parsed { + statements: vec![Expression::Null(Null)], + gaps: Vec::new(), + }), } } diff --git a/crates/sqlbuild-analysis/src/analysis_route/types.rs b/crates/sqlbuild-analysis/src/analysis_route/types.rs index fedc80ad24..9accb9c990 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/types.rs @@ -1,5 +1,8 @@ //! What a dialect's native analysis module provides. +use sqlquack_syntax::{Expression, Parsed}; + +use crate::analysis_route::constants::{STATEMENTS_CONSTRUCT, UNPARSED_STATEMENT_CONSTRUCT}; use crate::analysis_route::models::ParserGap; /// A dialect whose SQL SQLQuack analyses in place of Polyglot. @@ -7,8 +10,30 @@ pub trait NativeDialect: Sync { /// The adapter's analysis dialect name, such as `postgres`. fn name(&self) -> &'static str; - /// Whether the native parser reads `sql` whole: one statement, no gap and no unparsed bytes. - fn read_whole(&self, sql: &str) -> Result<(), ParserGap>; + /// Every statement of `sql` with the parser's named gaps, or the first error. + fn parse(&self, sql: &str) -> Result; + + /// The one statement of `sql`, with no gap and not a lone unparsed command. + fn read_whole(&self, sql: &str) -> Result { + let mut parsed: Parsed = self.parse(sql)?; + if let Some(gap) = parsed.gaps.first() { + return Err(ParserGap { + construct: gap.construct.to_owned(), + offset: gap.start, + }); + } + let unread = |construct: &str| ParserGap { + construct: construct.to_owned(), + offset: sql.len(), + }; + if parsed.statements.len() != 1 { + return Err(unread(STATEMENTS_CONSTRUCT)); + } + match parsed.statements.remove(0) { + Expression::Command(_) => Err(unread(UNPARSED_STATEMENT_CONSTRUCT)), + statement => Ok(statement), + } + } /// The engine-accepted prefix of UDF and cursor stubs: `__sqlbuild_`, or `__sqlbuild.` for T-SQL. fn stub_prefix(&self) -> &'static str; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs index c553e213e1..112d2aff10 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -1,14 +1,54 @@ -//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. +//! Parse one raw diff query with the dialect's parser and read its shape: SQLQuack where native, +//! else as the Polyglot wheel's guarded `parse_one` does. use polyglot_sql::dialects::Dialect; use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; +use sqlquack_syntax::traversal::types::ExpressionWalk as NativeWalk; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; use crate::diff_selection::models::RawQueryShape; -/// The query's shape, or the text of the exception the wheel raises for it. +/// The query's shape, or why it is not one statement the dialect's parser reads whole. pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { + match analysis_route(Some(dialect)) { + AnalysisRoute::Native(native) => { + let root: sqlquack_syntax::Expression = native + .read_whole(sql) + .map_err(|gap| format!("the {dialect} parser cannot read {}", gap.construct))?; + native_shape(&root) + } + AnalysisRoute::Legacy => polyglot_shape(sql, dialect), + } +} + +/// The shape of a statement SQLQuack read whole. +fn native_shape(root: &sqlquack_syntax::Expression) -> Result { + let root_key: String = root.variant_name().to_lowercase(); + let node_keys: Vec = if RAW_QUERY_READ_ONLY_ROOTS.contains(&root_key.as_str()) { + NativeWalk::dfs(root) + .map(|node| node.variant_name().to_lowercase()) + .collect() + } else { + Vec::new() + }; + let writes_or_locks: bool = match root { + sqlquack_syntax::Expression::Select(select) => { + select.into.is_some() || !select.locks.is_empty() + } + _ => false, + }; + Ok(RawQueryShape { + root_key, + node_keys, + writes_or_locks, + }) +} + +/// The shape Polyglot reads, or the text of the exception the wheel raises. +fn polyglot_shape(sql: &str, dialect: &str) -> Result { let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; let guard: ComplexityGuardOptions = diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs index 5afda5f19d..c3848a541a 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -7,7 +7,12 @@ use polyglot_sql::{ AnalyzeQueryOptions, DialectType, SchemaColumn, SchemaTable, ValidationSchema, analyze_query, }; use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlquack_syntax::Expression as NativeExpression; +use sqlquack_syntax::traversal::types::ExpressionWalk as NativeWalk; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::analysis_route::types::NativeDialect; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; use crate::planner_fixtures::models::FixtureStarRelation; @@ -32,6 +37,9 @@ pub fn star_keys( { return Vec::new(); } + if let AnalysisRoute::Native(native) = analysis_route(dialect) { + return native_star_keys(native, sql, relations); + } let Ok(dialect) = DialectType::from_str(dialect.filter(|name| !name.is_empty()).unwrap_or("generic")) else { @@ -123,3 +131,66 @@ fn column_type(sql_type: Option<&str>) -> String { .unwrap_or("UNKNOWN") .to_owned() } + +/// Star dependencies from SQLQuack's tree, a superset of what each star reads: a bare star selects +/// every relation in the query, and a star whose qualifier the tree does not settle, or SQL the +/// parser cannot read whole, selects every relation given. +fn native_star_keys( + native: &dyn NativeDialect, + sql: &str, + relations: &[FixtureStarRelation], +) -> Vec<(String, String)> { + let every = || { + relations + .iter() + .map(|relation| relation.key.clone()) + .collect() + }; + let Ok(sql) = stub_markers(sql) else { + return every(); + }; + let Ok(root) = native.read_whole(&sql) else { + return every(); + }; + let mut aliases: HashMap = HashMap::new(); + let mut stars: Vec> = Vec::new(); + for node in NativeWalk::dfs(&root) { + match node { + NativeExpression::Table(table) => { + let name: String = python_casefold(&table.name.name); + aliases.insert(name.clone(), name.clone()); + if let Some(alias) = &table.alias { + aliases.insert(python_casefold(&alias.name), name); + } + } + NativeExpression::Select(select) => { + stars.extend(select.expressions.iter().filter_map(|projection| { + match projection { + NativeExpression::Star(star) => Some( + star.table + .as_ref() + .map(|table| python_casefold(&table.name)), + ), + _ => None, + } + })); + } + _ => {} + } + } + let mut names: HashSet = HashSet::new(); + for star in &stars { + match star.as_ref().map(|qualifier| aliases.get(qualifier)) { + Some(Some(name)) => { + names.insert(name.clone()); + } + None => names.extend(aliases.values().cloned()), + Some(None) => return every(), + } + } + relations + .iter() + .filter(|relation| names.contains(&python_casefold(&relation.key.1))) + .map(|relation| relation.key.clone()) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs index 338624014a..1df5d5f24a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/fingerprint.rs @@ -1,11 +1,14 @@ //! Normalize migration metadata and SQL without changing the persisted format. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::planner_changes::main::version_identity_payload::version_identity_payload; use crate::planner_changes::models::IdentityPayload; use crate::planner_migrations::_helpers::proofs::ast::strip_formatting; +use crate::planner_migrations::_helpers::proofs::encoding::encode; use crate::planner_migrations::_helpers::proofs::local_names::{local_names, rewrite, unambiguous}; use crate::planner_migrations::_helpers::proofs::parse::parse; -use crate::planner_migrations::constants::{EXCLUDED_CONFIG, REF_MARKER}; +use crate::planner_migrations::constants::{EXCLUDED_CONFIG, NATIVE_TREE_PREFIX, REF_MARKER}; use polyglot_sql::{Expression, dialects::Dialect}; use regex::Regex; use serde_json::Value; @@ -74,6 +77,9 @@ pub(crate) fn normalized( } } rewrite(&mut parsed, &names, &ctes); + if let AnalysisRoute::Native(_) = analysis_route(Some(dialect)) { + return encode(&parsed).map(|tree| Some(format!("{NATIVE_TREE_PREFIX}{tree}"))); + } let expression: Expression = serde_json::from_value(parsed).map_err(|error| error.to_string())?; Dialect::get_by_name(dialect) diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs index a84b3ee534..f21c678c3a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs @@ -1,11 +1,46 @@ -//! Parse migration SQL under the compiler's complexity limit. +//! Parse migration SQL with the dialect's parser: SQLQuack where native, else Polyglot under the +//! compiler's complexity limit. +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::planner_migrations::constants::{SET_OPERATION_LEFT, WITH_KEY}; use polyglot_sql::dialects::Dialect; use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; use serde_json::{Value, json}; +/// The one statement of `sql` as a tree, or None when there is no proof: the parser cannot read it +/// whole, which callers take as the conservative plan. pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { + let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) else { + return polyglot_parse(sql, dialect); + }; + let Ok(statement) = native.read_whole(sql) else { + return Ok(None); + }; + let mut tree: Value = serde_json::to_value(statement).map_err(|error| error.to_string())?; + hoist_leading_with(&mut tree); + Ok(Some(tree)) +} + +/// Move a set operation's leading WITH from its first SELECT, where ClickHouse's parser hangs it, +/// onto the set operation, where the proofs read a statement's CTEs. +fn hoist_leading_with(tree: &mut Value) { + let Some(operation) = tree + .as_object_mut() + .and_then(|object| object.values_mut().next()) + .filter(|operation| operation.get(SET_OPERATION_LEFT).is_some()) + else { + return; + }; + if !operation[WITH_KEY].is_null() { + return; + } + let with: Value = leading_with(&mut operation[SET_OPERATION_LEFT]); + operation[WITH_KEY] = with; +} + +fn polyglot_parse(sql: &str, dialect: &str) -> Result, String> { let dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("unknown dialect {dialect}"))?; let guard: ComplexityGuardOptions = @@ -24,3 +59,20 @@ pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { .map(Some) .map_err(|error| error.to_string()) } + +/// The WITH of the first SELECT under `node`, taken out of it; null when it has none. +fn leading_with(node: &mut Value) -> Value { + let Some(inner) = node + .as_object_mut() + .and_then(|object| object.values_mut().next()) + else { + return Value::Null; + }; + if inner.get(SET_OPERATION_LEFT).is_some() { + return leading_with(&mut inner[SET_OPERATION_LEFT]); + } + inner + .as_object_mut() + .and_then(|select| select.insert(WITH_KEY.to_owned(), Value::Null)) + .unwrap_or(Value::Null) +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs index b760634f78..1cd92ada6a 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/constants.rs @@ -24,6 +24,8 @@ pub(crate) const OPAQUE_TUPLE_FIELDS: &[&str] = &[ ]; pub(crate) const HAVING_MAX_KEY: &str = "having_max"; pub(crate) const WITH_KEY: &str = "with"; +/// A set operation's first branch. +pub(crate) const SET_OPERATION_LEFT: &str = "left"; pub(crate) const TABLE_KEY: &str = "table"; pub(crate) const NUMBER_LITERAL: &str = "number"; @@ -49,3 +51,5 @@ pub(crate) const POLICY_FAIL: &str = "fail"; pub(crate) const MODEL_DONE: &str = "done"; pub(crate) const MODEL_RENAMED: &str = "renamed"; pub(crate) const MODEL_CONFLICT: &str = "conflict"; +/// Marks a native dialect's normalized SQL, its canonical tree, so it never equals Polyglot's SQL text. +pub(crate) const NATIVE_TREE_PREFIX: &str = "sqlquack-tree:"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs index aae5b546d0..9294a01917 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/_helpers/type_names.rs @@ -1,8 +1,12 @@ -//! Cursor types of warehouse type names and Polyglot type arguments. +//! Cursor types of warehouse type names and parsed type arguments. use serde_json::{Map, Value}; +use sqlquack_syntax::traversal::types::ExpressionWalk; +use sqlquack_syntax::{Expression, ParseOptions}; -use crate::planner_strategy::constants::{INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES}; +use crate::planner_strategy::constants::{ + INTEGER_TYPE_NAMES, TIMESTAMP_TYPE_NAMES, TYPE_PARSE_DIALECT, +}; use crate::planner_strategy::models::CursorType; /// The cursor type of a known type name. @@ -25,3 +29,28 @@ pub(crate) fn argument(args: &Map, key: &str) -> String { Some(other) => other.to_string().to_uppercase(), } } + +/// The scalar arguments of `type_sql` as SQLQuack's parser reads it in a cast, or why not. +pub(crate) fn type_args(type_sql: &str) -> Result, String> { + let statements: Vec = sqlquack_syntax::parse_strict( + &format!("SELECT CAST(NULL AS {type_sql})"), + TYPE_PARSE_DIALECT, + &ParseOptions::default(), + ) + .map_err(|error| error.to_string())?; + let data_type = statements + .iter() + .flat_map(|statement| statement.dfs()) + .find_map(|node| match node { + Expression::Cast(cast) => Some(&cast.to), + _ => None, + }) + .ok_or_else(|| format!("not one type: {type_sql}"))?; + match serde_json::to_value(data_type).map_err(|error| error.to_string())? { + Value::Object(fields) => Ok(fields + .into_iter() + .filter(|(_, value)| !matches!(value, Value::Object(_) | Value::Array(_))) + .collect()), + other => Err(format!("not one type: {other}")), + } +} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs index 1176dc46a1..ad15667c5d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/constants.rs @@ -72,10 +72,10 @@ pub const INTEGER_TYPE_NAMES: [&str; 17] = [ ]; /// Decimal type names whose zero-scale form is an integer cursor. pub const DECIMAL_TYPE_NAMES: [&str; 3] = ["DECIMAL", "NUMERIC", "NUMBER"]; -/// The type Polyglot reports for a type it does not know. -pub const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; -/// The dialect cursor type classification parses unknown types with. -pub const GENERIC_DIALECT: &str = "generic"; +/// The type the parser reports for a type it does not know. +pub const CUSTOM_TYPE_NAME: &str = "CUSTOM"; +/// The parser cursor type classification reads an unknown warehouse type with. +pub(crate) const TYPE_PARSE_DIALECT: sqlquack_syntax::Dialect = sqlquack_syntax::Dialect::DuckDB; /// Error code of an authored cursor input column its input does not expose. pub const CURSOR_INPUT_COLUMN_ERROR_CODE: &str = "S302"; /// Incremental strategies whose microbatches replay one batch by default. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs index 2be7a8dd5b..f0b4a53575 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/errors.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/errors.rs @@ -84,7 +84,7 @@ impl CappedWatermarkInput { } } -/// A warehouse type Polyglot could not parse, which classifies as no cursor type. +/// A warehouse type the parser could not read, which classifies as no cursor type. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TypeParseFailure { pub normalized_type: String, diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs index 22dedf16bd..4853f55f17 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/classify_cursor_sql_type.rs @@ -3,13 +3,12 @@ use serde_json::{Map, Value}; use crate::planner_strategy::_helpers::decimal_scale::decimal_scale_is_zero; -use crate::planner_strategy::_helpers::type_names::{argument, named_type}; -use crate::planner_strategy::constants::{GENERIC_DIALECT, POLYGLOT_CUSTOM_TYPE_NAME}; +use crate::planner_strategy::_helpers::type_names::{argument, named_type, type_args}; +use crate::planner_strategy::constants::CUSTOM_TYPE_NAME; use crate::planner_strategy::errors::TypeParseFailure; use crate::planner_strategy::models::CursorType; -use crate::type_system::main::polyglot_type_args::polyglot_type_args; -/// Timestamp or integer for supported cross-adapter types; `Err` when Polyglot cannot parse an unknown type. +/// Timestamp or integer for supported cross-adapter types; `Err` when the parser cannot read an unknown type. pub fn classify_cursor_sql_type(sql_type: &str) -> Result, TypeParseFailure> { let normalized: String = sql_type .to_uppercase() @@ -28,13 +27,12 @@ pub fn classify_cursor_sql_type(sql_type: &str) -> Result, Ty if let Some(cursor_type) = named_type(&base) { return Ok(Some(cursor_type)); } - let args: Map = - polyglot_type_args(&normalized, GENERIC_DIALECT).map_err(|message| TypeParseFailure { - normalized_type: normalized.clone(), - message, - })?; + let args: Map = type_args(&normalized).map_err(|message| TypeParseFailure { + normalized_type: normalized.clone(), + message, + })?; let mut type_name: String = argument(&args, "data_type"); - if type_name == POLYGLOT_CUSTOM_TYPE_NAME { + if type_name == CUSTOM_TYPE_NAME { type_name = argument(&args, "name").replace('_', ""); } Ok(named_type(&type_name)) diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs index d7b05b6719..9293db1051 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/cursor_type_warning.rs @@ -11,7 +11,7 @@ fn cursor_type_name(cursor_type: CursorType) -> &'static str { } } -/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any Polyglot parse error that left it unclassified. +/// An error (with SQL analysis) or warning when the warehouse type is the other cursor type, and any parse error that left it unclassified. pub fn cursor_type_warning( check: &CursorTypeCheck, ) -> (Option, Option) { diff --git a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs index 28fe14df79..277325f0fb 100644 --- a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs @@ -4,6 +4,8 @@ use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; use serde_json::{Map, Value, json}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; use crate::relation_names::constants::{ CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, NAME_KEY, QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, @@ -19,14 +21,12 @@ pub(crate) fn extracted_relations( let dialect: &str = dialect .filter(|name| !name.is_empty()) .unwrap_or(GENERIC_DIALECT); - let Some(statements) = parsed_statements(sql, dialect)? else { + let Some(trees) = statement_trees(sql, dialect)? else { return Ok(None); }; let mut extracted: ExtractedRelations = ExtractedRelations::default(); - for statement in &statements { - let tree: Value = serde_json::to_value(statement) - .map_err(|error| RelationNamesFailure::Internal(error.to_string()))?; - let nodes: Vec<&Map> = walked(&tree); + for tree in &trees { + let nodes: Vec<&Map> = walked(tree); let cte_names: Vec = nodes.iter().flat_map(|node| cte_names(node)).collect(); extracted .temporary @@ -45,6 +45,35 @@ pub(crate) fn extracted_relations( Ok(Some(extracted)) } +/// Each statement as a tree: SQLQuack's where native, with any gap or error None; else Polyglot's. +fn statement_trees(sql: &str, dialect: &str) -> Result>, RelationNamesFailure> { + let internal = |error: serde_json::Error| RelationNamesFailure::Internal(error.to_string()); + if let AnalysisRoute::Native(native) = analysis_route(Some(dialect)) { + if sql.trim().is_empty() { + return Ok(Some(Vec::new())); + } + return match native.parse(sql) { + Ok(parsed) if parsed.gaps.is_empty() => parsed + .statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal), + _ => Ok(None), + }; + } + let Some(statements) = parsed_statements(sql, dialect)? else { + return Ok(None); + }; + statements + .iter() + .map(serde_json::to_value) + .collect::, _>>() + .map(Some) + .map_err(internal) +} + /// The wheel's guarded `parse`: Ok(None) is the `PolyglotError` Python catches. fn parsed_statements( sql: &str, diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs index 7e49d9c53d..a9dea2d44e 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs @@ -2,7 +2,11 @@ use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; -use crate::snowflake_cost::constants::{CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER}; +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; +use crate::snowflake_cost::constants::{ + CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER, SNOWFLAKE_DIALECT, +}; pub(crate) fn of_kind(query_type: &str, kind: &str) -> bool { query_type == kind @@ -16,8 +20,14 @@ pub(crate) fn python_space(character: char) -> bool { character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) } -/// Whether a CREATE statement's text parses as a definition that needs no warehouse. +/// Whether a CREATE statement's text parses as a definition that needs no warehouse; false, the +/// warehouse-billed reading, when the parser cannot read it whole. pub(crate) fn metadata_only_create(query_text: &str) -> bool { + if let AnalysisRoute::Native(native) = analysis_route(Some(SNOWFLAKE_DIALECT)) { + return native + .read_whole(query_text) + .is_ok_and(|statement| native_metadata_only(&statement)); + } let options: ParseOptions = ParseOptions { complexity_guard: None, }; @@ -41,3 +51,18 @@ pub(crate) fn metadata_only_create(query_text: &str) -> bool { statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), } } + +/// Whether a statement SQLQuack read is a CREATE that needs no warehouse. +fn native_metadata_only(statement: &sqlquack_syntax::Expression) -> bool { + match statement { + sqlquack_syntax::Expression::CreateTable(table) => { + !(table.as_select.is_some() + || table.clone_source.is_some() + || table.using_template.is_some() + || table.table_modifier.as_deref().unwrap_or("").to_uppercase() + == DYNAMIC_TABLE_MODIFIER) + } + sqlquack_syntax::Expression::CreateView(view) if view.materialized => false, + statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs index 67a32cb1f1..9eae6ee8d0 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -8,3 +8,5 @@ pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; pub const QUERY_TAG_APP: &str = "sqlbuild"; pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; +/// Snowflake's analysis dialect name, which chooses its parser. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index 48f55a3095..840b93dcea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::records::ids::RecordId; const PLANNER_LOGGER: &str = "sqlbuild.planner"; -/// Log the Polyglot parse error that left a warehouse type unclassified. +/// Log the parse error that left a warehouse type unclassified. fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> { let text = |value: &str| PyString::new(py, value).into_any(); call_with(