diff --git a/Cargo.lock b/Cargo.lock index 5cf9f8a246..a2731dcd03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 9be0a1e595..c5cb6bad94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 129c9ec5ce..757a894a90 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -1,7 +1,5 @@ //! Per-model analysis cache keys over everything an analysis reads, and stored outcomes. -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::models::DeclaredOutput; use std::collections::{BTreeSet, HashMap}; use sha2::{Digest, Sha256}; @@ -171,20 +169,6 @@ impl KeyHasher { } self.text(&model.pivot_sql); self.families(&model.dynamic_families); - let declared: Vec = sql_macro_declared_outputs(&model.sql_macro_checks); - self.count(declared.len()); - for output in &declared { - match &output.cte { - Some(cte) => { - self.flag(true); - self.text(cte); - } - None => self.flag(false), - } - self.text(&output.name); - self.text(&output.returns); - self.count(output.branch); - } } fn families(&mut self, families: &[DynamicFamily]) { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index 74685b243c..dfdaceb69f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -8,10 +8,10 @@ use polyglot_sql::traversal::ExpressionWalk; use serde_json::{Map, Value}; use crate::assembly::analysis_session::_helpers::dict_walk::{ - PyValue, alias_or_name, arg, children, column_table_name, direct_tables, expression_args, - find_all, is_star, kind, name, name_payload, output_name, parse_one, projected_expression, - py_truthy, python_int, raw_name_payload, set_operation, to_dict, top_level_ctes, - unwrap_annotations, where_clause, + Assertion, PyValue, alias_or_name, arg, children, column_table_name, direct_tables, + expression_args, find_all, is_star, kind, name, name_payload, output_name, parse_one, + projected_expression, py_truthy, python_int, raw_name_payload, set_operation, to_dict, + top_level_ctes, typed_annotation, unwrap_annotations, where_clause, }; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::constants::{ @@ -23,7 +23,7 @@ use crate::assembly::analysis_session::constants::{ NULLABLE_NULLABILITY, NULLIF_FUNCTION_NAME, POLYGLOT_TYPE_NAMES, PYTHON_RULE, SAFE_CAST_RULE, SELECT_AST_KIND, SET_OPERATION_AST_KINDS, STRING_LITERAL_TYPE, SUBSTRING_AST_KIND, TABLE_AST_KIND, TEXT_TYPE, TIMESTAMP_TYPE_NAME, TIMESTAMP_TZ_TYPE, TRY_CAST_AST_KIND, - TYPE_PASSTHROUGH_AST_KINDS, UNKNOWN_NULLABILITY, VARCHAR_DATA_TYPES, WILDCARD, + TYPE_PASSTHROUGH_AST_KINDS, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, VARCHAR_DATA_TYPES, WILDCARD, }; use crate::assembly::analysis_session::constants::{ CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, TRANSFORM_AGGREGATION, TRANSFORM_CAST, @@ -35,8 +35,6 @@ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::models::DeclaredOutput; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeFamily}; @@ -60,8 +58,6 @@ pub(crate) struct RecoveryProfile<'a> { pub(crate) rules: Option<&'a Pairs>, /// Runs the adapter's own rules, those with the `python` rule id. pub(crate) callback: Option<&'a NullabilityCallback>, - /// The model's select items that are one SQL macro call each, typed by its `returns`. - pub(crate) declared: &'a [DeclaredOutput], } /// One enrichment's recovery request. @@ -133,108 +129,6 @@ pub(crate) struct LegacyAnalysis { pub(crate) lineage: Vec, } -/// The model query's output types its SQL macro call items change, given the relation types. -pub(crate) fn declared_output_types( - cleaned_sql: &str, - types: &ShapeTable, - profile: &RecoveryProfile<'_>, -) -> Fact> { - let Some(parsed) = parse_one(cleaned_sql, profile.dialect)? else { - return Ok(Vec::new()); - }; - let facts: Relations = referenced_types(&parsed, types); - let declared: Vec<(String, String)> = Context { - profile, - branches: declared_branches(&parsed), - } - .query_types(&parsed, &facts)?; - let plain: Vec<(String, String)> = Context { - profile, - branches: Vec::new(), - } - .query_types(&parsed, &facts)?; - let main: HashSet = main_output_columns(&parsed, profile.declared); - Ok(declared - .into_iter() - .filter(|(name, data_type)| { - dict_get(&plain, name) != Some(data_type) || main.contains(&casefold(name)) - }) - .collect()) -} - -/// Each select of the main query and top-level CTEs by address, with its scope and branch. -fn declared_branches(root: &Expression) -> Vec { - let mut branches: Vec = Vec::new(); - for (cte_name, has_column_aliases, body) in top_level_ctes(root) { - if !has_column_aliases - && unwrap_annotations(Some(body)).is_some_and(|body| top_level_ctes(body).is_empty()) - { - for (branch, select) in set_branches(Some(body)).into_iter().enumerate() { - branches.push(( - std::ptr::from_ref(select), - Some(cte_name.to_owned()), - branch, - )); - } - } - } - for (branch, select) in set_branches(Some(root)).into_iter().enumerate() { - branches.push((std::ptr::from_ref(select), None, branch)); - } - branches -} - -/// The folded main-query output columns its SQL macro call items stand in, in any branch. -fn main_output_columns(root: &Expression, declared: &[DeclaredOutput]) -> HashSet { - let selects: Vec<&Expression> = set_branches(Some(root)); - let by_name: bool = set_operation(root).is_some_and(|(_, _, by_name)| by_name); - let Some(first) = selects.first() else { - return HashSet::new(); - }; - let first_names: Vec<&str> = projection_names(first); - let mut columns: HashSet = HashSet::new(); - for output in declared.iter().filter(|output| output.cte.is_none()) { - if by_name { - columns.insert(casefold(&output.name)); - continue; - } - let Some(select) = selects.get(output.branch) else { - continue; - }; - let position: Option = projection_names(select) - .iter() - .position(|name| name.eq_ignore_ascii_case(&output.name)); - if let Some(column) = position.and_then(|position| first_names.get(position)) { - columns.insert(casefold(column)); - } - } - columns -} - -/// The output name of each of `select`'s projections, in order. -fn projection_names(select: &Expression) -> Vec<&str> { - select - .get_expressions() - .iter() - .map(|projection| output_name(unwrap_annotations(Some(projection)))) - .collect() -} - -/// The selects of `node`'s set operation, left to right, or `node` itself. -fn set_branches(node: Option<&Expression>) -> Vec<&Expression> { - let Some(node) = unwrap_annotations(node) else { - return Vec::new(); - }; - match set_operation(node) { - Some((left, right, _)) => { - let mut branches: Vec<&Expression> = set_branches(Some(left)); - branches.extend(set_branches(Some(right))); - branches - } - None => vec![node], - } -} - /// Python's legacy analysis of a model the native engine handed back, or why it defers. pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { let Some(parsed) = parse_one(input.cleaned_sql, input.profile.dialect)? else { @@ -272,7 +166,6 @@ pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { let unqualified: Option<&(String, String)> = single_resource(&resources); let context = Context { profile: &input.profile, - branches: Vec::new(), }; let (cte_types, cte_nullability) = if input.recover { ( @@ -543,7 +436,6 @@ fn passthrough_facts( } let context = Context { profile: &input.profile, - branches: Vec::new(), }; recovery.types = context.passthrough_types(root, &input_types(input))?; recovery.nullability = context.passthrough_nullability(root, &input_nullability(input))?; @@ -574,21 +466,24 @@ fn direct_cte_outputs(root: &Expression, cte_names: &HashSet) -> HashSet outputs } -/// A select's address, its top-level CTE or `None` for the main query, and its branch. -type Branch = (*const Expression, Option, usize); - /// The profile every resolver reads. struct Context<'a, 'b> { profile: &'a RecoveryProfile<'b>, - /// The selects whose SQL macro call items take their declared types. - branches: Vec, } impl Context<'_, '_> { - /// The model query's output types, through its top-level CTEs. - fn query_types(&self, root: &Expression, facts: &Relations) -> Fact> { + /// Python's `_polyglot_cte_passthrough_types_from_parsed`. + fn passthrough_types( + &self, + root: &Expression, + facts: &Relations, + ) -> Fact> { + let ctes: Vec<(&str, bool, &Expression)> = top_level_ctes(root); + if ctes.is_empty() { + return Ok(Vec::new()); + } let mut relations: Relations = referenced_facts(root, facts); - for (cte_name, has_column_aliases, body) in top_level_ctes(root) { + for (cte_name, has_column_aliases, body) in ctes { if has_column_aliases { continue; } @@ -604,52 +499,6 @@ impl Context<'_, '_> { self.direct_select_types(Some(root), &relations) } - /// Python's `_polyglot_cte_passthrough_types_from_parsed`. - fn passthrough_types( - &self, - root: &Expression, - facts: &Relations, - ) -> Fact> { - if top_level_ctes(root).is_empty() { - return Ok(Vec::new()); - } - self.query_types(root, facts) - } - - /// `types` of `select` with each of its SQL macro call items typed by its `returns`. - fn declared_types( - &self, - select: &Expression, - mut types: Vec<(String, String)>, - ) -> Vec<(String, String)> { - let address: *const Expression = std::ptr::from_ref(select); - let Some((_, cte, branch)) = self - .branches - .iter() - .find(|(branch_select, _, _)| *branch_select == address) - else { - return types; - }; - for output in self.profile.declared { - let key: String = types - .iter() - .map(|(name, _)| name) - .find(|name| name.eq_ignore_ascii_case(&output.name)) - .cloned() - .unwrap_or_else(|| output.name.clone()); - let inferred: Option<&str> = dict_get(&types, &key).map(String::as_str); - if let Some(declared) = sql_macro_declared_type( - self.profile.declared, - (cte.as_deref(), *branch, &output.name), - inferred, - self.profile.dialect, - ) { - types = dict_set(types, &key, declared); - } - } - types - } - /// Python's `_polyglot_select_output_types`. fn select_output_types( &self, @@ -840,7 +689,7 @@ impl Context<'_, '_> { inferred = dict_set(inferred, output, inferred_type); } } - Ok(self.declared_types(select, inferred)) + Ok(inferred) } /// Python's `_polyglot_direct_expression_type`. @@ -849,6 +698,24 @@ impl Context<'_, '_> { expression: Option<&Expression>, by_alias: &Relations, ) -> Fact> { + if let Some((asserted, inner)) = typed_annotation(expression) { + if let Some(inferred) = self + .expression_type_in(Some(inner), by_alias)? + .filter(|inferred| !upper(inferred).contains(UNKNOWN_TYPE)) + { + return Ok(Some(inferred)); + } + return match asserted { + Assertion::Fixed(asserted) => { + let target: Value = + serde_json::to_value(asserted).map_err(|error| error.to_string())?; + Ok(target.as_object().and_then(cast_target_type)) + } + Assertion::Like(like) => Ok(self + .expression_type_in(Some(like), by_alias)? + .filter(|like| !upper(like).contains(UNKNOWN_TYPE))), + }; + } let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); if let Some(inferred) = self.expression_type(expression)? { @@ -1070,6 +937,7 @@ impl Context<'_, '_> { aliases: &[(String, &'static str)], scoped: &NullRelations, ) -> Fact<&'static str> { + let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); match node_kind { NULL_AST_KIND => return Ok(NULLABLE_NULLABILITY), @@ -1101,6 +969,7 @@ impl Context<'_, '_> { /// Python's `_infer_polyglot_shallow_nullability`. fn shallow_nullability(&self, expression: Option<&Expression>) -> Fact<&'static str> { + let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); match node_kind { NULL_AST_KIND => return Ok(NULLABLE_NULLABILITY), diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs index 287eebd81f..bbdf207c06 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs @@ -2,7 +2,7 @@ use polyglot_sql::expressions::Cte; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions, ast_json}; +use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions, ast_json}; use serde_json::{Map, Value, json}; use crate::assembly::analysis_session::models::ProjectionFacts; @@ -254,6 +254,28 @@ pub(crate) fn unwrap_annotations(mut node: Option<&Expression>) -> Option<&Expre node } +/// What a type assertion asserts: a type, or the type of another expression. +pub(crate) enum Assertion<'a> { + Fixed(&'a DataType), + Like(&'a Expression), +} + +/// The first type assertion around `node`, past plain annotations, and the expression it types. +pub(crate) fn typed_annotation( + mut node: Option<&Expression>, +) -> Option<(Assertion<'_>, &Expression)> { + while let Some(Expression::Annotated(annotated)) = node { + if let Some(asserted) = &annotated.type_assertion { + return Some((Assertion::Fixed(asserted), &annotated.this)); + } + if let Some(like) = &annotated.type_like { + return Some((Assertion::Like(like), &annotated.this)); + } + node = Some(&annotated.this); + } + None +} + /// A projection's expression: an alias's `this`, otherwise the projection. pub(crate) fn projected_expression(projection: Option<&Expression>) -> Option<&Expression> { if kind(projection) == ALIAS_AST_KIND { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index cefd99d2b9..2d69ab47a5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -113,7 +113,6 @@ fn legacy(input: &EnrichmentInput<'_>, cleaned: &str) -> Result = input diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs index f360b6b503..7a20a27572 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs @@ -182,7 +182,6 @@ impl Tables<'_> { function_return_types: &self.request.function_return_types, rules: self.request.nullability_rules.as_ref(), callback: self.request.nullability_callback.as_ref(), - declared: &[], }, }) })?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index d8d1e7d87a..19327a8c92 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -19,7 +19,7 @@ use crate::assembly::analysis_session::_helpers::compact_batch::{ Batch, BatchMember, MemberAnalysis, MemberResult, }; use crate::assembly::analysis_session::_helpers::cte_facts::{ - LegacyAnalysis, LegacyInput, RecoveryProfile, declared_output_types, legacy_analysis, + LegacyAnalysis, LegacyInput, RecoveryProfile, legacy_analysis, }; use crate::assembly::analysis_session::_helpers::dynamic_pivot::{PivotFacts, pivot_outcome}; use crate::assembly::analysis_session::_helpers::enrichment::{ @@ -40,9 +40,7 @@ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments; -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::models::{DeclaredOutput, SqlMacroCheck, SqlMacroFinding}; +use crate::sql_macros::models::SqlMacroFinding; use crate::sql_macros::types::ArgumentCheckInput; /// What an internal native failure of the session names. @@ -296,76 +294,6 @@ impl AnalysisSession { self.check_sql_macros(models) } - /// Type the outputs SQL macro call items reach by the macros' `returns`, through CTEs. - fn type_sql_macro_outputs(&mut self, models: &[usize]) -> Result<(), String> { - for model in models { - let declared: Vec = - sql_macro_declared_outputs(&self.request.models[*model].sql_macro_checks); - if declared.is_empty() || self.cache_hit(*model) { - continue; - } - self.ensure_cleaned(&[*model])?; - let Some(outcome) = self.outcomes[*model].as_ref() else { - continue; - }; - let Ok(cleaned) = resolved_cleaned_sql(outcome) else { - continue; - }; - let profile = RecoveryProfile { - dialect: &self.request.dialect, - function_return_types: &self.request.function_return_types, - rules: self.request.nullability_rules.as_ref(), - callback: self.request.nullability_callback.as_ref(), - declared: &declared, - }; - let propagated: Result, String> = catch_compiler_panic(|| { - Ok(declared_output_types( - &cleaned, - &self.available_types, - &profile, - )) - }) - .map_err(|reason| { - native_failure(&model_context(&self.request.models[*model]), &reason) - })?; - let Ok(typed) = propagated else { - continue; - }; - let dialect: String = self.request.dialect.clone(); - let Some(columns) = self.outcome_mut(*model)?.analysis.columns.as_mut() else { - continue; - }; - for column in columns { - let Some((name, returns)) = typed - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(&column.name)) - else { - continue; - }; - let reached: [DeclaredOutput; 1] = [DeclaredOutput { - cte: None, - name: name.clone(), - returns: returns.clone(), - branch: 0, - }]; - if let Some(data_type) = sql_macro_declared_type( - &reached, - (None, 0, &column.name), - column.data_type.as_deref(), - &dialect, - ) { - column.data_type = Some(data_type); - } - } - } - Ok(()) - } - - /// Whether the cache answered `model` with an outcome that already holds its declared types. - fn cache_hit(&self, model: usize) -> bool { - self.cache.as_ref().is_some_and(|cache| cache.hits[model]) - } - /// Each model's SQL macro call arguments checked against its analysed types, cache hits too. fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> { let checked: Vec = models @@ -373,7 +301,7 @@ impl AnalysisSession { .copied() .filter(|model| { let request: &ModelRequest = &self.request.models[*model]; - checks_arguments(&request.sql_macro_checks) && request.placeholders.is_empty() + !request.sql_macro_checks.is_empty() && request.placeholders.is_empty() }) .collect(); self.ensure_cleaned(&checked)?; @@ -427,7 +355,6 @@ impl AnalysisSession { function_return_types: &self.request.function_return_types, rules: self.request.nullability_rules.as_ref(), callback: self.request.nullability_callback.as_ref(), - declared: &[], }, }) }) @@ -459,7 +386,10 @@ impl AnalysisSession { } let request: &ModelRequest = &self.request.models[model]; let required: Vec = request.required_names.clone(); - let has_set_operation: bool = request.has_set_operation; + let unmarked_set_operation: bool = request.has_set_operation + && !request + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); let inputs_known: bool = !required.is_empty() && required .iter() @@ -479,14 +409,25 @@ impl AnalysisSession { .iter() .flatten() .any(|column| column.data_type.is_none()); - if !required.is_empty() && (untyped || star_pending) && !has_set_operation { + let marked: bool = inputs_known + && self.request.models[model] + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); + let analysis: &ModelAnalysis = &self.outcome_mut(model)?.analysis; + if !required.is_empty() + && (untyped || star_pending || marked && analysis.analysis_succeeded) + && !unmarked_set_operation + { candidates.push((model, input_schemas, star_pending)); } } let enrichments: Vec = self.native_enrichments(&candidates)?; for ((model, _, star_pending), native) in candidates.into_iter().zip(enrichments) { + let marked: bool = self.request.models[model] + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); let outcome: &mut ModelOutcome = self.outcome_mut(model)?; - outcome.analysis = enriched_analysis(&outcome.analysis, native, star_pending); + outcome.analysis = enriched_analysis(&outcome.analysis, native, (star_pending, marked)); } Ok(()) } @@ -549,7 +490,6 @@ impl AnalysisSession { /// Python's deferred binding validation, then the dataflow's shape publication. fn finish_wave(&mut self, wave: usize) -> Result<(), String> { let models: Vec = self.waves[wave].clone(); - self.type_sql_macro_outputs(&models)?; let validated: &[usize] = if self.cyclic { &[] } else { &models }; let mut revalidated: Vec<(usize, Shapes)> = Vec::new(); for model in validated { @@ -795,11 +735,6 @@ impl AnalysisSession { } } -/// Whether any of the checks has arguments the analysed SQL must show fit. -fn checks_arguments(checks: &[SqlMacroCheck]) -> bool { - checks.iter().any(|check| !check.arguments.is_empty()) -} - fn batch_member<'a>(model: &'a ModelRequest, schema: &'a Shapes) -> BatchMember<'a> { BatchMember { query_sql: &model.query_sql, @@ -888,7 +823,7 @@ fn legacy_outcome( fn enriched_analysis( analysis: &ModelAnalysis, enriched: Enrichment, - star_pending: bool, + (star_pending, marked): (bool, bool), ) -> ModelAnalysis { let mut recovered: HashMap = HashMap::new(); for column in enriched @@ -901,6 +836,16 @@ fn enriched_analysis( recovered.insert(column.name.clone(), data_type.clone()); } } + let retyped: HashMap = if marked && enriched.analysis_succeeded { + enriched + .columns + .iter() + .flatten() + .filter_map(|column| Some((column.name.clone(), column.data_type.clone()?))) + .collect() + } else { + HashMap::new() + }; let star_expanded: bool = star_pending && enriched.analysis_succeeded; let replaced: bool = (!has_columns(analysis.columns.as_ref()) || star_expanded) && enriched.analysis_succeeded; @@ -919,7 +864,9 @@ fn enriched_analysis( }; let mut columns: Vec = merged.columns.take().unwrap_or_default(); for column in &mut columns { - if column.data_type.as_deref().is_none_or(str::is_empty) { + if let Some(data_type) = retyped.get(&column.name) { + column.data_type = Some(data_type.clone()); + } else if column.data_type.as_deref().is_none_or(str::is_empty) { column.data_type = recovered.get(&column.name).cloned(); } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 2b786482a0..0cd68c4162 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -196,7 +196,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v2"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v3"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs index 27a9755d87..b65c4a0ad7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs @@ -66,7 +66,6 @@ pub fn expression_shapes( function_return_types: &request.function_return_types, rules: request.nullability_rules.as_ref(), callback: request.nullability_callback.as_ref(), - declared: &[], }, }) })?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs index acd8cc734a..92b9f05478 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs @@ -29,7 +29,6 @@ fn recovered_facts(request: &CteFactRequest) -> Result { function_return_types: &request.function_return_types, rules: request.nullability_rules.as_ref(), callback: None, - declared: &[], }, })?; let mut direct_outputs: Vec = recovered.direct_outputs.into_iter().collect(); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index 95de9dfe5c..6cb3eb54b3 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -22,3 +22,5 @@ mod native_dialect; mod reference_facts; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_typed_markers.rs"] +mod typed_markers; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 49eb329990..8205ca8924 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -27,7 +27,6 @@ use crate::assembly::analysis_session::tests::test_types::{ }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::{CatalogInput, ProjectCatalog}; -use crate::sql_macros::models::{CallScope, SqlMacroCheck, TypedOutput}; const TRANSFORM_NAMES: [&str; 6] = [ "direct", @@ -180,11 +179,6 @@ pub(crate) fn cached_run(request: SessionRequest, store: NativeStore) -> CachedR (keys, hits, taken) }); CachedRun { - deferred: outcome - .models - .iter() - .map(|model| model.deferred_cleaning.is_some()) - .collect(), lines: ( steps.iter().map(step_lines).collect(), outcome.models.iter().map(described).collect(), @@ -477,7 +471,6 @@ pub(crate) fn recovered_facts(sql: &str) -> Option<(Pairs, Pairs, Vec, V function_return_types: &Vec::new(), rules: Some(&rules), callback: None, - declared: &[], }, }) .ok()?; @@ -557,7 +550,6 @@ pub(crate) fn legacy_lines(sql: &str) -> Option> { function_return_types: &Vec::new(), rules: Some(&rules), callback: None, - declared: &[], }, }) .ok()?; @@ -706,20 +698,29 @@ pub(crate) fn injected_relations( request } -/// The orders request over `models`, the first typing `label` in CTE `final` as VARCHAR. -pub(crate) fn declared_label_request(models: &[ModelSpec]) -> SessionRequest { - let mut requests: Vec = model_requests(models); - requests[0].sql_macro_checks.push(SqlMacroCheck { - macro_name: "label".to_owned(), - scope: CallScope::Cte("final".to_owned()), - plain_select: true, - arguments: Vec::new(), - call: None, - output: Some(TypedOutput { - name: "label".to_owned(), - returns: "VARCHAR".to_owned(), - branch: 0, - }), - }); - orders_request(requests) +/// The request a fast-lineage contract compile sends for `typed` over `sql`, reading `staged`. +pub(crate) fn typed_marker_request(sql: &'static str) -> SessionRequest { + let staged: ModelSpec = ( + "staged", + "SELECT * FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ); + let mut requests = model_requests(&[("typed", sql, &[], &["staged"]), staged]); + requests[0].recover_cte_facts = true; + let mut request: SessionRequest = orders_request(requests); + request.rich_type_inference = false; + let own = shapes(&[("typed", &[("v", "VARCHAR")])]); + request.column_types.extend(own.clone()); + request.complete_schemas.extend(own); + request +} + +/// The described line of the first model's column `v`. +pub(crate) fn described_column_v(lines: &[Vec]) -> String { + lines[0] + .iter() + .find(|line| line.starts_with("v ")) + .cloned() + .unwrap_or_else(|| format!("no v in {:?}", lines[0])) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs index 914908700f..24c55a4b51 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs @@ -3,13 +3,13 @@ use sqlbuild_cache::store::models::NativeStore; use crate::assembly::analysis_session::models::{ModelReference, SessionRequest}; use crate::assembly::analysis_session::tests::helpers::{ - amounts_family, cache_stats, cached_run, completed, current_model_key, damaged, - declared_label_request, described, model_keys, model_requests, orders_request, pairs, - second_model, session_lines, shapes, started, step_lines, uncached_catalog, + amounts_family, cache_stats, cached_run, current_model_key, damaged, model_keys, + model_requests, orders_request, pairs, second_model, session_lines, shapes, started, + uncached_catalog, }; use crate::assembly::analysis_session::tests::test_types::{ CacheEditTestCase, CacheKeyTestCase, CacheReuseTestCase, CachedRun, DamagedCacheTestCase, - DeclaredHitTestCase, ModelSpec, RelationDigestTestCase, + ModelSpec, RelationDigestTestCase, }; const STG_ORDERS: ModelSpec = ( @@ -73,15 +73,6 @@ const EVENTS_TOO_DEEP: ModelSpec = ( &[], ); -/// Labels orders with a function the inference does not type, in CTE `final`. -const LABELLED: ModelSpec = ( - "labelled", - "WITH final AS (SELECT order_id, bar(amount, 0, 10) AS label FROM __source(\"raw_orders\")) \ - SELECT * FROM final", - &["raw_orders"], - &[], -); - #[test] fn given_a_filled_cache_when_rerunning_then_hits_match_the_uncached_session() { let test_cases = [ @@ -528,47 +519,3 @@ fn given_a_read_relation_fact_changed_when_keying_then_the_reading_model_key_cha ); } } - -#[test] -fn given_a_declared_sql_macro_output_when_rerunning_then_the_hit_keeps_its_type_unpatched() { - let test_cases = [DeclaredHitTestCase { - description: "a call item in a final CTE, read with a star", - models: &[LABELLED], - expected_label: "label VARCHAR", - }]; - for test_case in test_cases { - let (uncached_steps, uncached_models, _) = - completed(started(declared_label_request(test_case.models))); - let cold: CachedRun = cached_run( - declared_label_request(test_case.models), - NativeStore::default(), - ); - let cold_lines = cold.lines; - let warm: CachedRun = cached_run(declared_label_request(test_case.models), cold.store); - let uncached = ( - uncached_steps.iter().map(step_lines).collect::>(), - uncached_models.iter().map(described).collect::>(), - ); - - assert!( - uncached.1[0] - .iter() - .any(|line| line.starts_with(test_case.expected_label)), - "{}: {:?}", - test_case.description, - uncached.1 - ); - assert_eq!( - (&cold_lines, &warm.lines), - (&uncached, &uncached), - "{}", - test_case.description - ); - assert_eq!( - (warm.hits, warm.deferred), - (vec![true], vec![true]), - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs new file mode 100644 index 0000000000..8a8a270e5c --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs @@ -0,0 +1,104 @@ +use sqlbuild_cache::store::models::NativeStore; + +use crate::assembly::analysis_session::tests::helpers::{ + cached_run, completed, described, described_column_v, started, typed_marker_request, +}; +use crate::assembly::analysis_session::tests::test_types::{ + CachedRun, TypedMarkerCacheTestCase, TypedMarkerTestCase, +}; + +#[test] +fn given_typed_markers_when_analysing_then_each_call_has_its_asserted_or_inferred_type() { + let test_cases = [ + TypedMarkerTestCase { + description: "an unknown body takes the asserted type", + sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a known body keeps its inferred type", + sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v INT ", + }, + TypedMarkerTestCase { + description: "a known aliased column keeps its inferred type", + sql: "SELECT __polyglot_typed(o.order_id, 'VARCHAR') AS v FROM __ref(\"staged\") AS o", + expected_column: "v INT ", + }, + TypedMarkerTestCase { + description: "a later set-operation branch types the union", + sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ + SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a NULL body takes the asserted type in a union", + sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"staged\") AS o UNION ALL \ + SELECT __polyglot_typed(NULL, 'BIGINT') AS v FROM __ref(\"staged\") AS o", + expected_column: "v BIGINT", + }, + TypedMarkerTestCase { + description: "a CTE read by name", + sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ + FROM __ref(\"staged\")) SELECT v FROM c", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "an operand of a larger expression", + sql: "SELECT COALESCE(__polyglot_typed(bar(amount, 0, 10), 'VARCHAR'), 'x') AS v \ + FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + ]; + + for test_case in test_cases { + let (_, outcomes, _) = completed(started(typed_marker_request(test_case.sql))); + let lines: Vec> = outcomes.iter().map(described).collect(); + let column: String = described_column_v(&lines); + assert!( + column.starts_with(test_case.expected_column), + "{}: {column}", + test_case.description, + ); + } +} + +#[test] +fn given_typed_markers_when_rerunning_from_the_cache_then_the_hit_matches_the_cold_run() { + let test_cases = [ + TypedMarkerCacheTestCase { + description: "an unknown body takes the asserted type", + sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a known body keeps its inferred type", + sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a later set-operation branch types the union", + sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ + SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a CTE read by name", + sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ + FROM __ref(\"staged\")) SELECT v FROM c", + expected_hits: &[true, true], + }, + ]; + + for test_case in test_cases { + let cold: CachedRun = + cached_run(typed_marker_request(test_case.sql), NativeStore::default()); + let warm: CachedRun = cached_run(typed_marker_request(test_case.sql), cold.store); + assert_eq!(warm.lines, cold.lines, "{}", test_case.description); + assert_eq!( + warm.hits, test_case.expected_hits, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 9568fadd92..aa29f89c8d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -144,8 +144,6 @@ pub(crate) struct CachedRun { pub(crate) keys: Vec>, /// Whether each model's outcome came from the cache. pub(crate) hits: Vec, - /// Whether each model's outcome still defers its cleaned SQL. - pub(crate) deferred: Vec, /// The catalog's schema additions and analysis names, then the sorted fact models. pub(crate) catalog: ( crate::assembly::analysis_session::types::Shapes, @@ -220,9 +218,15 @@ pub(crate) struct DeltaSessionTestCase { pub(crate) expected_whole_session_outcome: bool, } -/// Models whose first one declares a SQL macro output, and that output's described column. -pub(crate) struct DeclaredHitTestCase { +/// A model query holding typed markers, and the described line of its column `v`. +pub(crate) struct TypedMarkerTestCase { pub(crate) description: &'static str, - pub(crate) models: &'static [ModelSpec], - pub(crate) expected_label: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_column: &'static str, +} + +pub(crate) struct TypedMarkerCacheTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_hits: &'static [bool], } diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs index e0f79d49e6..3f6e1724f9 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs @@ -39,5 +39,6 @@ pub(crate) fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { enum_declarations: model.enum_declarations, constant_declarations: model.constant_declarations, analysis: None, + typed_calls: model.typed_calls, } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index d769f01a51..309221dca8 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -1,5 +1,6 @@ //! Facts a compile's producers retain about each resource, in production order. +use crate::sql_macros::models::TypedCall; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; @@ -31,6 +32,8 @@ pub struct CompiledModelFacts { pub constant_declarations: Vec, /// The typed analysis result; `None` until the assembly stage transfers it. pub analysis: Option, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A model's effective config values with sorted keys, as the rules request has always read them. @@ -212,6 +215,8 @@ pub struct PlainModel { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A one-based line and column, counted in characters. diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index eacc966c75..8603bf2d2d 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -109,6 +109,7 @@ pub(crate) fn model_contract_diagnostics( contract_enforced, unchecked: &unchecked, types, + untyped_calls: &model.untyped_call_outputs, }; diagnostics.extend(column_type(&checked, &check)?); } @@ -120,6 +121,8 @@ struct TypeCheck<'a> { contract_enforced: bool, unchecked: &'a HashSet<&'a str>, types: &'a TypeComparer<'a>, + /// Output columns that are one call to a SQL macro without `returns`, and the macro. + untyped_calls: &'a [(String, String)], } struct TypedColumn<'a> { @@ -269,7 +272,17 @@ fn column_type( column_name: name.to_owned(), message: UNPROVEN_TYPE_OUTPUT_MESSAGE.to_owned(), }), - help: UNKNOWN_TYPE_HELP.to_owned(), + help: match check + .untyped_calls + .iter() + .find(|(output, _)| output.eq_ignore_ascii_case(name)) + { + Some((_, macro_name)) => format!( + "declare returns {declared_type} in SQL macro {macro_name}(), or add an \ + explicit CAST if this declared type should be checked statically" + ), + None => UNKNOWN_TYPE_HELP.to_owned(), + }, })); }; if check.types.types_equal(declared_type, inferred_type)? == TypeComparison::Equal { diff --git a/crates/sqlbuild-analysis/src/contracts/models.rs b/crates/sqlbuild-analysis/src/contracts/models.rs index 656f80975a..868b8904eb 100644 --- a/crates/sqlbuild-analysis/src/contracts/models.rs +++ b/crates/sqlbuild-analysis/src/contracts/models.rs @@ -23,6 +23,8 @@ pub struct ContractModel { pub fast_lineage_has_star: bool, pub dynamic_proof: Option, pub unchecked_output_columns: Vec, + /// Output columns that are one call to a SQL macro without `returns`, with the macro's name. + pub untyped_call_outputs: Vec<(String, String)>, } /// The declared shape of one model. diff --git a/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs b/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs index e573e13ec7..bb4ff5d21a 100644 --- a/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs @@ -12,6 +12,7 @@ pub(crate) fn model(contract: Option<&str>, schema: Option) -> C fast_lineage_has_star: false, dynamic_proof: None, unchecked_output_columns: Vec::new(), + untyped_call_outputs: Vec::new(), } } diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs index 7fdce06669..18a35b2a60 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs @@ -7,7 +7,10 @@ use crate::sql_macros::models::{ParameterKind, SqlMacroParameter}; pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String { let kind: String = match ¶meter.kind { ParameterKind::Expression { data_type } => format!("expression {}", type_text(data_type)), - ParameterKind::Any => "expression any".to_owned(), + ParameterKind::Any { family: None } => "expression any".to_owned(), + ParameterKind::Any { + family: Some(family), + } => format!("expression any {}", family.keyword().to_lowercase()), ParameterKind::Constant { data_type } => format!("const {}", type_text(data_type)), ParameterKind::Collection { set, data_type } => { format!("collection {set} {}", type_text(data_type)) diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs index d06562d4dc..5c63caad55 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs @@ -12,11 +12,11 @@ pub fn sql_macro_definition_digest(name: &str, path: &str, source: &str) -> Stri }; let mut parts: Vec = vec![ definition.form.label().to_owned(), - definition - .returns - .as_deref() - .map(type_text) - .unwrap_or_default(), + match (&definition.returns, &definition.returns_like) { + (Some(returns), _) => type_text(returns), + (None, Some(like)) => format!("type of {}", like.to_lowercase()), + (None, None) => String::new(), + }, sql_definition_digest(&definition.body), ]; for parameter in &definition.parameters { diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs new file mode 100644 index 0000000000..28a8d6b503 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs @@ -0,0 +1,47 @@ +//! Facts of a fully annotated query: which calls annotation could not type, and output types. + +use polyglot_sql::expressions::DataType; +use polyglot_sql::optimizer::annotate_types::function_result_is_known; +use polyglot_sql::traversal::ExpressionWalk; +use polyglot_sql::{DialectType, Expression}; + +/// Whether a call under `node`, outside a typed marker, has an unknown or only guessed type. +pub(crate) fn has_untyped_call(node: &Expression, dialect: DialectType) -> bool { + match node { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, + Expression::Function(function) + if !function_result_is_known(function, dialect) || is_unknown(node) => + { + true + } + Expression::AggregateFunction(_) | Expression::WindowFunction(_) if is_unknown(node) => { + true + } + _ => node + .children() + .into_iter() + .any(|child| has_untyped_call(child, dialect)), + } +} + +/// Whether annotation left `node` without a type, or with a wholly unknown one. +fn is_unknown(node: &Expression) -> bool { + node.inferred_type() + .is_none_or(|data_type| *data_type == DataType::Unknown) +} + +/// The annotated type of each projection of a `SELECT`, past its alias. +pub(crate) fn projection_types(expression: &Expression) -> Vec> { + let Expression::Select(select) = expression else { + return Vec::new(); + }; + let mut types: Vec> = Vec::with_capacity(select.expressions.len()); + for projection in &select.expressions { + let value: &Expression = match projection { + Expression::Alias(alias) => &alias.this, + other => other, + }; + types.push(value.inferred_type().cloned()); + } + types +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index e029f9af3e..ad8781052c 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -280,7 +280,10 @@ impl TypeContext<'_> { } fn projection_type(&self, expression: &Expression, aliases: &Sources) -> Option { - if matches!(unwrapped(expression), Expression::Null(_)) { + if asserted_expression(expression).is_none() + && unresolved_like(expression).is_none() + && matches!(unwrapped(expression), Expression::Null(_)) + { Some(NULL_TYPE.to_string()) } else { self.expression(expression, aliases).or_else(|| { @@ -294,6 +297,21 @@ impl TypeContext<'_> { } fn expression(&self, expression: &Expression, aliases: &Sources) -> Option { + if let Some((like, inner)) = unresolved_like(expression) { + let known = |data_type: &String| !data_type.to_ascii_uppercase().contains("UNKNOWN"); + return self + .expression(inner, aliases) + .filter(|inferred| known(inferred) && inferred != NULL_TYPE) + .or_else(|| self.expression(like, aliases).filter(known)); + } + if let Some((asserted, inner)) = asserted_expression_type(expression) { + if matches!(unwrapped(inner), Expression::Null(_)) { + return self.cast_type(asserted); + } + return self + .expression(inner, aliases) + .filter(|inferred| !inferred.to_ascii_uppercase().contains("UNKNOWN")); + } let expression = unwrapped(expression); let function_name = match expression { Expression::Function(function) => function.name.as_str(), @@ -435,6 +453,45 @@ fn query_with(expression: &Expression) -> Option<&With> { } } +/// The expression a type assertion around `expression` types, past parentheses and comments. +fn asserted_expression(expression: &Expression) -> Option<&Expression> { + asserted_expression_type(expression).map(|(_, inner)| inner) +} + +/// The type assertion around `expression`, past parentheses and comments, and what it types. +fn asserted_expression_type(mut expression: &Expression) -> Option<(&DataType, &Expression)> { + loop { + expression = match expression { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => { + return annotation + .type_assertion + .as_ref() + .map(|asserted| (asserted, &annotation.this)); + } + Expression::Annotated(annotation) => &annotation.this, + Expression::Paren(paren) => &paren.this, + _ => return None, + }; + } +} + +/// The like and typed expressions of an unresolved typed-like assertion wrapping `expression`. +fn unresolved_like(mut expression: &Expression) -> Option<(&Expression, &Expression)> { + loop { + expression = match expression { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => { + return None; + } + Expression::Annotated(annotation) => match &annotation.type_like { + Some(like) => return Some((like, &annotation.this)), + None => &annotation.this, + }, + Expression::Paren(paren) => &paren.this, + _ => return None, + }; + } +} + fn unwrapped(mut expression: &Expression) -> &Expression { loop { expression = match expression { diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs new file mode 100644 index 0000000000..45e65c7c99 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs @@ -0,0 +1,55 @@ +use std::collections::HashMap; + +use polyglot_sql::expressions::DataType; +use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; + +use crate::query_analysis::annotated_types::{has_untyped_call, projection_types}; +use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_outputs}; +use crate::query_analysis::types::AnnotatedOutput; + +/// Each output of the one query in `sql`, typed by full annotation; `None` if a call is untyped. +#[must_use] +pub fn annotated_output_types( + sql: &str, + dialect: &str, + schema: &ValidationSchema, +) -> Option> { + let dialect: DialectType = match dialect.parse() { + Ok(dialect) => dialect, + Err(_) => return None, + }; + let mut statements: Vec = + match polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()) { + Ok(statements) => statements, + Err(_) => return None, + }; + if statements.len() != 1 { + return None; + } + let mut expression: Expression = statements.pop()?; + let mapping = polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( + schema, dialect, + ); + polyglot_sql::optimizer::annotate_types::annotate_types( + &mut expression, + Some(&mapping as &dyn polyglot_sql::schema::Schema), + Some(dialect), + ); + if has_untyped_call(&expression, dialect) { + return None; + } + let annotated: Vec> = projection_types(&expression); + let mut outputs: Vec = Vec::new(); + for (index, (name, data_type)) in + infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) + .into_iter() + .enumerate() + { + outputs.push(( + name, + data_type.filter(|value| value != NULL_TYPE), + annotated.get(index).cloned().flatten(), + )); + } + Some(outputs) +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs index 2634257679..f51bc838b0 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -1,5 +1,6 @@ pub mod analyze; pub mod analyze_project; pub mod analyze_project_compact; +pub mod annotated_output_types; pub mod output_types; pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-analysis/src/query_analysis/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/mod.rs index 94f93ea625..847eabad9a 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/mod.rs @@ -1,3 +1,5 @@ +#[path = "_helpers/annotated_types.rs"] +mod annotated_types; #[path = "_helpers/borrowed_facts.rs"] mod borrowed_facts; #[path = "_helpers/compatibility_types.rs"] @@ -5,6 +7,7 @@ mod compatibility_types; #[path = "_helpers/engine.rs"] mod engine; pub mod main; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/query_analysis/types.rs b/crates/sqlbuild-analysis/src/query_analysis/types.rs new file mode 100644 index 0000000000..ff1b963f81 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/types.rs @@ -0,0 +1,6 @@ +//! Shapes of query analysis results. + +use polyglot_sql::expressions::DataType; + +/// One output of a fully annotated query: its name, inferred type, and annotated type. +pub type AnnotatedOutput = (String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs index c7557c5132..b2ed75855c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod arguments; -pub(crate) mod body; -pub(crate) mod definition; -pub(crate) mod expansion; -pub(crate) mod header; -pub(crate) mod lexing; -pub(crate) mod literal_positions; -pub(crate) mod parameters; -pub(crate) mod placement; -pub(crate) mod probes; +pub(crate) mod parsing; +pub(crate) mod rendering; +pub(crate) mod typing; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs similarity index 78% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs index 46bfffd2b0..9a45512806 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs @@ -3,17 +3,18 @@ use polyglot_sql::expressions::{Identifier, Select}; use polyglot_sql::{DialectType, Expression, ExpressionWalk}; -use crate::sql_macros::_helpers::arguments::is_atomic; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::parameters::{ +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::parameters::{ ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, with_placeholders, }; +use crate::sql_macros::_helpers::rendering::arguments::is_atomic; +use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentShape, BodyExpansion, ParameterKind, RenderedArgument, SqlMacroDefinition, - SqlMacroForm, SqlMacroParameter, + ArgumentShape, BodyExpansion, ExpandedText, ParameterKind, RenderedArgument, + SqlMacroDefinition, SqlMacroForm, SqlMacroParameter, TextSplice, TypedCall, }; /// Dialects tried in turn to parse a body; the first that parses it locates its references. @@ -215,7 +216,9 @@ fn substituted( _ => argument.sql.clone(), } } - ParameterKind::Expression { .. } | ParameterKind::Any | ParameterKind::Constant { .. } + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::Constant { .. } if !is_atomic(&argument.sql) => { format!("({})", argument.sql) @@ -224,13 +227,15 @@ fn substituted( } } -/// `body` with each `@name` replaced by its argument, token by token. +/// `body` with each `@name` replaced by its argument, token by token, and the calls it holds. fn substituted_body( definition: &SqlMacroDefinition, (body, scanned, references): (&str, &[Token], &[ParameterReference]), - arguments: &[RenderedArgument], -) -> String { + (arguments, body_calls): (&[RenderedArgument], &[TypedCall]), +) -> (String, Vec) { let mut expanded: String = String::with_capacity(body.len()); + let mut splices: Vec = Vec::new(); + let mut argument_calls: Vec = Vec::new(); let mut cursor: usize = 0; for reference in references { let Some(argument) = arguments.iter().find(|argument| { @@ -242,24 +247,58 @@ fn substituted_body( }; let parameter = &definition.parameters[argument.parameter]; expanded.push_str(&body[cursor..reference.start]); - expanded.push_str(&substituted( + let text: String = substituted( ¶meter.kind, ¶meter.name, argument, (body, scanned, reference.token), - )); + ); + let start: usize = expanded.len(); + if !argument.calls.is_empty() { + let inner: Option = if text == argument.sql { + Some(0) + } else if text.len() == argument.sql.len() + 2 && text[1..].starts_with(&argument.sql) { + Some(1) + } else { + None + }; + if let Some(inner) = inner { + argument_calls.extend(shifted(argument.calls.clone(), start + inner)); + } + } + expanded.push_str(&text); + splices.push(TextSplice { + source: (reference.start, reference.end), + output: (start, expanded.len()), + }); cursor = reference.end; } expanded.push_str(&body[cursor..]); - expanded + let mut calls: Vec = argument_calls; + calls.extend(spliced(body_calls.to_vec(), &splices)); + (expanded, calls) +} + +/// `calls` in `text` kept within its `range`, relative to the range's start. +fn within(calls: Vec, (start, end): (usize, usize)) -> Vec { + calls + .into_iter() + .filter(|call| start <= call.start && call.start < end) + .map(|call| TypedCall { + start: call.start - start, + end: call.end.min(end) - start, + ..call + }) + .collect() } /// `body` with each parameter replaced by its argument, as the form inserts it at a call. pub(crate) fn expand_body( definition: &SqlMacroDefinition, - body: &str, + body: &ExpandedText, arguments: &[RenderedArgument], ) -> Result { + let (body, body_calls): (&str, &[TypedCall]) = (&body.sql, &body.calls); let invalid = |reason: String| { SqlMacroError::new( INVALID_DEFINITION_CODE, @@ -286,12 +325,28 @@ pub(crate) fn expand_body( format!("macro {}(): {reason}", definition.name), ) })?; - let expanded: String = substituted_body(definition, (body, &scanned, &references), arguments); - let sql: String = match definition.form { - SqlMacroForm::Expression => expanded.trim().to_owned(), + let (expanded, calls) = substituted_body( + definition, + (body, &scanned, &references), + (arguments, body_calls), + ); + let kept: (usize, usize) = match definition.form { + SqlMacroForm::Expression => trimmed_range(&expanded, 0), SqlMacroForm::CteList | SqlMacroForm::ColumnList => without_leading_keyword(&expanded), }; - Ok(BodyExpansion { sql, names }) + Ok(BodyExpansion { + sql: expanded[kept.0..kept.1].to_owned(), + names, + calls: within(calls, kept), + like: None, + }) +} + +/// The byte range of `text[from..]` trimmed of surrounding whitespace. +fn trimmed_range(text: &str, from: usize) -> (usize, usize) { + let rest: &str = &text[from..]; + let start: usize = from + rest.len() - rest.trim_start().len(); + (start, start + rest.trim().len()) } /// `sql`, with a line break when it ends in a `--` comment and `followed` text would join that line. @@ -327,13 +382,13 @@ pub(crate) fn with_line_comment_closed(sql: &str, followed: bool) -> String { sql } -/// The text after its first token, the body's `WITH` or `SELECT`, trimmed. -fn without_leading_keyword(text: &str) -> String { +/// The byte range of the text after its first token, the body's `WITH` or `SELECT`, trimmed. +fn without_leading_keyword(text: &str) -> (usize, usize) { match tokens(text) { Ok(scanned) => match scanned.first() { - Some(first) => text[first.end..].trim().to_owned(), - None => text.trim().to_owned(), + Some(first) => trimmed_range(text, first.end), + None => trimmed_range(text, 0), }, - Err(_) => text.trim().to_owned(), + Err(_) => trimmed_range(text, 0), } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs similarity index 83% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs index 0631d5a5af..c401284890 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs @@ -1,13 +1,15 @@ //! A SQL macro file parsed into its definition, then checked on its own. -use crate::sql_macros::_helpers::arguments::{ShapeContext, argument_shape, check_shape}; -use crate::sql_macros::_helpers::header::{Header, HeaderParser}; -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs}; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, line_column, tokens}; -use crate::sql_macros::_helpers::literal_positions::{ +use crate::sql_macros::_helpers::parsing::header::{Header, HeaderParser}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; -use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::rendering::arguments::{ + ShapeContext, argument_shape, check_shape, +}; use crate::sql_macros::constants::{ BARE_MACRO_CALL_CODE, BODY_REFERENCE_CODE, INVALID_DEFINITION_CODE, MIXED_KIND_CODE, PARAMETER_REFERENCE_CODE, REFERENCE_FUNCTIONS, RESERVED_PARAMETER_NAMES, SELECT_KEYWORD, @@ -15,7 +17,7 @@ use crate::sql_macros::constants::{ }; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ParameterKind, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, + ParameterKind, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, SqlMacroParameter, }; /// Locates a byte offset of the macro file. @@ -42,7 +44,17 @@ pub(crate) fn parsed_definition( let form: SqlMacroForm = body_form(body, &scanned); check_references(name, body, &scanned, &in_body)?; let calls: Vec = body_calls(name, body, &in_body)?; - check_returns(name, (form, header.returns.as_ref()), at(0))?; + check_returns( + name, + ( + form, + header.returns.is_some() || header.returns_like.is_some(), + ), + at(0), + )?; + if let Some((like, offset)) = &header.returns_like { + check_returns_like(name, (like, &header.parameters), at(*offset))?; + } let (body_line, body_column) = line_column(source, body_start); let definition: SqlMacroDefinition = SqlMacroDefinition { name: name.to_owned(), @@ -50,6 +62,7 @@ pub(crate) fn parsed_definition( description: header.description, parameters: header.parameters, returns: header.returns, + returns_like: header.returns_like.map(|(like, _)| like), form, body: body.to_owned(), body_line, @@ -154,20 +167,14 @@ fn body_calls(name: &str, body: &str, in_body: Locate<'_>) -> Result Ok(calls) } -/// Rejects an expression macro without `returns` and another form with it. +/// Rejects `returns` on a CTE-list or column-list macro. fn check_returns( name: &str, - (form, returns): (SqlMacroForm, Option<&String>), + (form, returns): (SqlMacroForm, bool), location: SqlMacroLocation, ) -> Result<(), SqlMacroError> { match (form, returns) { - (SqlMacroForm::Expression, None) => Err(SqlMacroError::new( - INVALID_DEFINITION_CODE, - format!("expression macro {name}() needs returns in its MACRO header"), - ) - .at(location) - .helped("declare the type the expression returns, such as returns DECIMAL(18, 2)")), - (SqlMacroForm::CteList | SqlMacroForm::ColumnList, Some(_)) => Err(SqlMacroError::new( + (SqlMacroForm::CteList | SqlMacroForm::ColumnList, true) => Err(SqlMacroError::new( INVALID_DEFINITION_CODE, format!( "{} macro {name}() cannot declare returns; only expression macros return a value", @@ -179,6 +186,50 @@ fn check_returns( } } +/// Rejects `returns TYPE OF` naming no argument, or an argument with no scalar type. +fn check_returns_like( + name: &str, + (like, parameters): (&str, &[SqlMacroParameter]), + location: SqlMacroLocation, +) -> Result<(), SqlMacroError> { + let Some(parameter) = parameters + .iter() + .find(|parameter| parameter.name.eq_ignore_ascii_case(like)) + else { + let names: Vec<&str> = parameters + .iter() + .map(|parameter| parameter.name.as_str()) + .collect(); + return Err(SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!("returns TYPE OF {like} names no argument of macro {name}()"), + ) + .at(location) + .helped(if names.is_empty() { + format!("macro {name}() has no arguments; declare returns instead") + } else { + format!("name one of its arguments: {}", names.join(", ")) + })); + }; + match parameter.kind { + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::Constant { .. } + | ParameterKind::Enum { .. } => Ok(()), + ParameterKind::Collection { .. } + | ParameterKind::Relation { .. } + | ParameterKind::ColumnOf { .. } => Err(SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!( + "returns TYPE OF {} names a {} argument of macro {name}(); only a scalar argument has a type to return", + parameter.name, + parameter.kind.label() + ), + ) + .at(location)), + } +} + /// Rejects an `@name` naming no parameter and a RELATION parameter written bare. fn check_parameter_references( definition: &SqlMacroDefinition, @@ -306,7 +357,9 @@ fn check_references( let referenced: String = scanned .get(index + 2) .filter(|argument| matches!(argument.kind, TokenKind::Quoted | TokenKind::Text)) - .map(|argument| crate::sql_macros::_helpers::header::unquoted(argument.text(body))) + .map(|argument| { + crate::sql_macros::_helpers::parsing::header::unquoted(argument.text(body)) + }) .unwrap_or_default(); let kind: &str = match function { "__source" => "source", diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs similarity index 89% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs index e644af2802..3ed9989d10 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs @@ -1,15 +1,17 @@ //! The `MACRO (...)` header of a SQL macro file: description, typed arguments and result type. -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{ + Token, TokenKind, closing_token, line_column, tokens, +}; use crate::sql_macros::constants::{ ANY_KEYWORD, ARGUMENTS_KEY, COLUMN_KEYWORD, COMMA, CONST_KEYWORD, DEFAULT_KEYWORD, DESCRIPTION_KEY, ENUM_KEYWORD, HEADER_KEYWORD, INVALID_DEFINITION_CODE, LIST_KEYWORD, OF_KEYWORD, RAW_TYPE_NAMES, RELATION_COLUMN_MIN_TOKENS, RELATION_KEYWORD, RETURNS_KEY, - SET_KEYWORD, + SET_KEYWORD, TYPE_KEYWORD, }; use crate::sql_macros::errors::LexError; use crate::sql_macros::errors::SqlMacroError; -use crate::sql_macros::models::{ParameterKind, SqlMacroLocation, SqlMacroParameter}; +use crate::sql_macros::models::{AnyFamily, ParameterKind, SqlMacroLocation, SqlMacroParameter}; use sqlbuild_sqltext::header_strings::constants::{ DOUBLE_QUOTE_INSIDE_HELP, DOUBLE_QUOTE_INSIDE_MESSAGE, SINGLE_QUOTE_INSIDE_HELP, SINGLE_QUOTE_INSIDE_MESSAGE, TRIPLE_QUOTE_INSIDE_HELP, TRIPLE_QUOTE_INSIDE_MESSAGE, @@ -22,6 +24,8 @@ pub(crate) struct Header { pub(crate) description: Option, pub(crate) parameters: Vec, pub(crate) returns: Option, + /// The argument named by `returns TYPE OF `, and its byte offset. + pub(crate) returns_like: Option<(String, usize)>, /// Byte offset just past the header and its optional `;`. pub(crate) body_start: usize, } @@ -83,6 +87,7 @@ impl HeaderParser<'_> { description: None, parameters: Vec::new(), returns: None, + returns_like: None, body_start, }; for entry in split_top_level(source, &scanned[2..close]) { @@ -145,6 +150,19 @@ impl HeaderParser<'_> { if value.is_empty() { return Err(self.error(key.start, "returns needs a SQL type")); } + if let [type_word, of, argument] = value + && type_word.is_word(source, TYPE_KEYWORD) + && of.is_word(source, OF_KEYWORD) + { + if argument.kind != TokenKind::Word { + return Err(self.error( + argument.start, + "returns TYPE OF needs an argument name, such as returns TYPE OF value", + )); + } + header.returns_like = Some((argument.text(source).to_owned(), argument.start)); + return Ok(header); + } header.returns = Some(type_text(source, value)); Ok(header) } @@ -288,22 +306,38 @@ impl HeaderParser<'_> { } if first.is_word(source, ANY_KEYWORD) { let Some(after) = tokens.get(1) else { - return Ok(ParameterKind::Any); + return Ok(ParameterKind::Any { family: None }); }; + let family: Option = AnyFamily::ALL + .into_iter() + .find(|family| after.is_word(source, family.keyword())); + if let Some(family) = family { + let Some(extra) = tokens.get(2) else { + return Ok(ParameterKind::Any { + family: Some(family), + }); + }; + return Err(self + .error(extra.start, format!("argument {name} ANY {} takes nothing after it; CONST, LIST and SET are not supported for ANY families", family.keyword())) + .helped(format!("declare it {name} ANY {}, or give it a SQL type", family.keyword()))); + } if after.is_word(source, CONST_KEYWORD) { return Err(self .error(after.start, format!("argument {name} ANY CONST: a CONST argument needs a type its values are checked against")) .helped(format!("declare it with the type of its values, such as {name} VARCHAR CONST"))); } return Err(self - .error(after.start, format!("argument {name} ANY takes nothing after it; ANY accepts any scalar expression")) - .helped(format!("declare it {name} ANY, or give it a SQL type"))); + .error(after.start, format!("argument {name} ANY takes nothing after it but NUMERIC, TEXT or TEMPORAL; ANY accepts any scalar expression")) + .helped(format!("declare it {name} ANY, or {name} ANY NUMERIC, ANY TEXT or ANY TEMPORAL for one type family, or give it a SQL type"))); } if first.is_word(source, LIST_KEYWORD) || first.is_word(source, SET_KEYWORD) { let set: bool = first.is_word(source, SET_KEYWORD); let keyword: String = first.text(source).to_ascii_uppercase(); let element: &[Token] = &tokens[1..]; - if element.len() == 1 && element[0].is_word(source, ANY_KEYWORD) { + if element + .first() + .is_some_and(|first| first.is_word(source, ANY_KEYWORD)) + { return Err(self .error( element[0].start, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs similarity index 100% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs similarity index 96% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs index 515eb037e5..839e030ed8 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs @@ -1,7 +1,7 @@ //! Parameters a SQL macro body uses where SQL requires a literal, such as `LIMIT @n`. -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token}; -use crate::sql_macros::_helpers::parameters::ParameterReference; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token}; +use crate::sql_macros::_helpers::parsing::parameters::ParameterReference; use crate::sql_macros::constants::{ LITERAL_REQUIRED_CODE, ROUND_FUNCTION, ROUND_PRECISION_ARGUMENT, }; @@ -39,7 +39,9 @@ pub(crate) fn non_literal_use<'r>( let parameter = definition.parameter(&reference.name)?; if !matches!( parameter.kind, - ParameterKind::Expression { .. } | ParameterKind::Any | ParameterKind::ColumnOf { .. } + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::ColumnOf { .. } ) { return None; } @@ -232,7 +234,7 @@ pub(crate) fn literal_error( Some(ParameterKind::Expression { data_type }) => { format!("declare it {name} {data_type} CONST") } - Some(ParameterKind::Any) => { + Some(ParameterKind::Any { .. }) => { format!("declare it with a type and CONST, such as {name} VARCHAR CONST") } _ => format!("pass the value through a CONST argument, such as {name}_value VARCHAR CONST"), diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs new file mode 100644 index 0000000000..7e98da38eb --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod body; +pub(crate) mod definition; +pub(crate) mod header; +pub(crate) mod lexing; +pub(crate) mod literal_positions; +pub(crate) mod parameters; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs similarity index 97% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs index 05039f6b00..d825103bb4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs @@ -2,7 +2,7 @@ use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; -use crate::sql_macros::_helpers::lexing::{DECLARATION_WORDS, Token, TokenKind}; +use crate::sql_macros::_helpers::parsing::lexing::{DECLARATION_WORDS, Token, TokenKind}; const PLACEHOLDER_PREFIX: &str = "__sqlbuild_parameter__"; /// Words that may follow a table reference without being its alias. diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs similarity index 94% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs index 76769a2286..4a32f7256f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs @@ -1,7 +1,8 @@ //! A call's argument list: split, bound to parameters and checked against their types. -use crate::sql_macros::_helpers::header::{split_top_level, unquoted}; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::parsing::header::{split_top_level, unquoted}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; use crate::sql_macros::constants::{ ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, }; @@ -238,7 +239,7 @@ pub(crate) fn argument_shape(text: &str) -> ArgumentShape { } fn unquote(text: &str) -> String { - crate::sql_macros::_helpers::header::unquoted(text) + crate::sql_macros::_helpers::parsing::header::unquoted(text) } /// The type family of a SQL literal argument. @@ -350,11 +351,34 @@ pub(crate) fn check_shape( Err(expects("a SQL expression")) } (ParameterKind::Expression { .. }, _) => Ok(()), - (ParameterKind::Any, ArgumentShape::InlineCollection | ArgumentShape::Reference { .. }) => { + ( + ParameterKind::Any { .. }, + ArgumentShape::InlineCollection | ArgumentShape::Reference { .. }, + ) => { Err(expects("a scalar SQL expression") .helped("an ANY argument takes one value per row, such as a column, a literal or an expression; pass lists to LIST arguments and relations to RELATION arguments")) } - (ParameterKind::Any, _) => Ok(()), + ( + ParameterKind::Any { + family: Some(family), + }, + ArgumentShape::Literal, + ) => { + let actual: TypeFamily = literal_family(argument); + if family.admits(actual) { + Ok(()) + } else { + Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {}; {shown} is {}", + family.noun(), + family_label(actual) + ), + )) + } + } + (ParameterKind::Any { .. }, _) => Ok(()), (ParameterKind::Constant { .. }, ArgumentShape::ConstantReference { .. }) => Ok(()), (ParameterKind::Constant { .. }, _) => Err(expects("a literal, a scalar constant or an enum value") .helped(format!("pass a literal, such as 2, or a constant, such as @const(\"{name}\"); a CONST argument is known at compile time, so a column or expression cannot be passed"))), @@ -391,10 +415,10 @@ pub(crate) fn check_shape( /// Whether an argument of kind `outer`, forwarded as `@name`, fits a parameter of kind `inner`. fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { match inner { - ParameterKind::Expression { .. } | ParameterKind::Any => matches!( + ParameterKind::Expression { .. } | ParameterKind::Any { .. } => matches!( outer, ParameterKind::Expression { .. } - | ParameterKind::Any + | ParameterKind::Any { .. } | ParameterKind::Constant { .. } | ParameterKind::Enum { .. } | ParameterKind::ColumnOf { .. } @@ -493,11 +517,7 @@ pub(crate) fn is_atomic(sql: &str) -> bool { /// The family of a declared type, normalized as schema declaration types are for the adapter. pub(crate) fn type_family(data_type: &str, adapter: &str) -> TypeFamily { use crate::type_system::models::TypeFamily as Normalized; - let dialect: &str = match adapter { - "motherduck" => "duckdb", - "" => "generic", - other => other, - }; + let dialect: &str = adapter_dialect(adapter); let normalized = normalize_type(data_type, dialect) .or_else(|_| normalize_type(data_type, "generic")) .map(|normalization| normalization.normalized.family); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs new file mode 100644 index 0000000000..709fcb84c9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs @@ -0,0 +1,382 @@ +//! A CTE-list macro body's CTEs renamed for one call's `__cte` name, located by parsing. + +use polyglot_sql::expressions::Identifier; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; + +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::parameters::{ + ParameterReference, parameter_references, placeholder, with_placeholders, +}; +use crate::sql_macros::_helpers::typing::typed_calls::spliced; +use crate::sql_macros::constants::{ARGUMENT_CODE, CTE_ARGUMENT, CTE_NAME_SEPARATOR}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + CallArgument, ExpandedText, SqlMacroDefinition, SqlMacroForm, TextSplice, +}; + +/// Dialects tried in turn to parse a body for its CTE names. +const PARSE_DIALECTS: [DialectType; 4] = [ + DialectType::Generic, + DialectType::DuckDB, + DialectType::Snowflake, + DialectType::BigQuery, +]; +/// The query a CTE list is completed with to parse it. +const CTE_SUFFIX: &str = "\nSELECT 1"; + +/// The call's `__cte` name, removed from its arguments, or why it cannot be used. +pub(crate) fn take_cte_argument( + definition: &SqlMacroDefinition, + arguments: Vec, +) -> Result<(Option, Vec), SqlMacroError> { + let mut name: Option = None; + let mut kept: Vec = Vec::with_capacity(arguments.len()); + for argument in arguments { + if !argument + .name + .as_deref() + .is_some_and(|named| named.eq_ignore_ascii_case(CTE_ARGUMENT)) + { + kept.push(argument); + continue; + } + if definition.form != SqlMacroForm::CteList { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} names the CTEs a CTE-list macro inserts; {}() is {} {} macro", + definition.name, + if definition.form == SqlMacroForm::Expression { + "an" + } else { + "a" + }, + definition.form.label() + ), + ) + .helped(format!("remove {CTE_ARGUMENT} := {}", argument.text.trim()))); + } + if name.is_some() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} is passed twice in a call to macro {}()", + definition.name + ), + )); + } + let text: &str = argument.text.trim(); + if !is_bare_identifier(text) { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} of macro {}() must be a bare CTE name; got {text}", + definition.name + ), + ) + .helped(format!("pass a name such as {CTE_ARGUMENT} := eu_orders"))); + } + name = Some(text.to_owned()); + } + Ok((name, kept)) +} + +/// Whether `text` is one unquoted identifier. +fn is_bare_identifier(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// The name each of a body's CTEs takes: the last takes `name`, the others `__`. +fn renamed(ctes: &[String], name: &str, cte: &str) -> Option { + let index: usize = ctes + .iter() + .position(|defined| defined.eq_ignore_ascii_case(cte))?; + Some(if index + 1 == ctes.len() { + name.to_owned() + } else { + format!("{name}{CTE_NAME_SEPARATOR}{}", ctes[index]) + }) +} + +/// `body` with every CTE it defines, and every reference to one, renamed for `name`. +pub(crate) fn renamed_ctes(body: &ExpandedText, name: &str) -> Result { + let sql: &str = &body.sql; + let scanned: Vec = + tokens(sql).map_err(|_| "it has an unclosed quote or comment".to_owned())?; + let references: Vec = parameter_references(sql, &scanned); + let parsed_text: String = format!("{}{CTE_SUFFIX}", with_placeholders(sql, &references)); + let statement: Expression = parsed(&parsed_text)?; + let Expression::Select(select) = &statement else { + return Err("it is not a WITH list".to_owned()); + }; + let Some(with) = &select.with else { + return Err("it is not a WITH list".to_owned()); + }; + let ctes: Vec = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); + let mut identifiers: Vec<&Identifier> = with.ctes.iter().map(|cte| &cte.alias).collect(); + let mut walk = CteReferences { + frames: Vec::new(), + scopes: Vec::new(), + found: Vec::new(), + }; + walk.visit(&statement, true); + identifiers.extend(walk.found); + let offsets: CharOffsets = CharOffsets::new(&parsed_text); + let mut edits: Vec<(usize, usize, String)> = Vec::new(); + for identifier in identifiers { + let Some(new_name) = renamed(&ctes, name, &identifier.name) else { + continue; + }; + let Some(span) = identifier.span else { + continue; + }; + let start: usize = original_offset(offsets.byte(span.start), sql, &references); + let end: usize = original_offset(offsets.byte(span.end), sql, &references); + let text: String = if identifier.quoted { + format!("\"{new_name}\"") + } else { + new_name + }; + edits.push((start, end, text)); + } + edits.sort_by_key(|(start, _, _)| *start); + edits.dedup_by_key(|(start, _, _)| *start); + let mut renamed_sql: String = String::with_capacity(sql.len() + edits.len() * name.len()); + let mut splices: Vec = Vec::with_capacity(edits.len()); + let mut cursor: usize = 0; + for (start, end, text) in edits { + if start < cursor || end > sql.len() { + continue; + } + renamed_sql.push_str(&sql[cursor..start]); + let output: usize = renamed_sql.len(); + renamed_sql.push_str(&text); + splices.push(TextSplice { + source: (start, end), + output: (output, renamed_sql.len()), + }); + cursor = end; + } + renamed_sql.push_str(&sql[cursor..]); + Ok(ExpandedText { + calls: spliced(body.calls.clone(), &splices), + sql: renamed_sql, + }) +} + +/// One relation a query reads by name: the name it is read as, and whether that is the body's CTE. +struct ScopeSource { + name: String, + body_cte: bool, +} + +/// The table names and qualifiers that read a body's own CTEs, resolved scope by scope. +struct CteReferences<'a> { + /// The CTE names of each enclosing `WITH`, innermost last, and whether they are the body's. + frames: Vec<(Vec, bool)>, + /// The relations of each enclosing query, innermost last. + scopes: Vec>, + found: Vec<&'a Identifier>, +} + +impl<'a> CteReferences<'a> { + fn visit(&mut self, node: &'a Expression, top: bool) { + let with = match node { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(union) => union.with.as_ref(), + Expression::Intersect(intersect) => intersect.with.as_ref(), + Expression::Except(except) => except.with.as_ref(), + _ => None, + }; + if let Some(with) = with { + let names: Vec = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); + for (index, cte) in with.ctes.iter().enumerate() { + let visible: usize = if with.recursive { index + 1 } else { index }; + self.frames.push((names[..visible].to_vec(), top)); + self.visit(&cte.this, false); + self.frames.pop(); + } + self.frames.push((names, top)); + } + let scoped: bool = if let Expression::Select(select) = node { + let mut sources: Vec = Vec::new(); + for source in select + .from + .iter() + .flat_map(|from| &from.expressions) + .chain(select.joins.iter().map(|join| &join.this)) + { + sources.extend(self.scope_sources(source)); + } + self.scopes.push(sources); + true + } else { + false + }; + match node { + Expression::Table(table) if self.is_body_cte(table) => self.found.push(&table.name), + Expression::Column(column) => self.qualifier(column.table.as_ref()), + Expression::Star(star) => self.qualifier(star.table.as_ref()), + _ => {} + } + for child in node.children() { + if !is_cte_body(with, child) { + self.visit(child, false); + } + } + if scoped { + self.scopes.pop(); + } + if with.is_some() { + self.frames.pop(); + } + } + + /// Whether `table` is an unqualified name whose innermost binding is one of the body's CTEs. + fn is_body_cte(&self, table: &polyglot_sql::expressions::TableRef) -> bool { + if table.schema.is_some() || table.catalog.is_some() { + return false; + } + self.frames + .iter() + .rev() + .find(|(names, _)| includes_name(names, &table.name.name)) + .is_some_and(|(_, body)| *body) + } + + /// Each name a FROM or JOIN item is read as, and whether it is an unaliased body CTE. + fn scope_sources(&self, source: &Expression) -> Vec { + let aliased = |alias: &Identifier| ScopeSource { + name: alias.name.clone(), + body_cte: false, + }; + match source { + Expression::Table(table) => vec![match &table.alias { + Some(alias) => aliased(alias), + None => ScopeSource { + name: table.name.name.clone(), + body_cte: self.is_body_cte(table), + }, + }], + Expression::Subquery(subquery) => subquery.alias.iter().map(aliased).collect(), + Expression::JoinedTable(joined) => match &joined.alias { + Some(alias) => vec![aliased(alias)], + None => { + let mut sources: Vec = self.scope_sources(&joined.left); + for join in &joined.joins { + sources.extend(self.scope_sources(&join.this)); + } + sources + } + }, + Expression::Paren(paren) => self.scope_sources(&paren.this), + _ => Vec::new(), + } + } + + /// Records `qualifier` when the relation it names, innermost scope first, is a body CTE. + fn qualifier(&mut self, qualifier: Option<&'a Identifier>) { + let Some(qualifier) = qualifier else { + return; + }; + let source: Option<&ScopeSource> = self + .scopes + .iter() + .rev() + .find_map(|sources| source_named(sources, &qualifier.name)); + if source.is_some_and(|source| source.body_cte) { + self.found.push(qualifier); + } + } +} + +/// Whether `child` is the body of one of `with`'s CTEs, which are visited in their own frames. +fn is_cte_body(with: Option<&polyglot_sql::expressions::With>, child: &Expression) -> bool { + let Some(with) = with else { + return false; + }; + for cte in &with.ctes { + if std::ptr::eq(&cte.this, child) { + return true; + } + } + false +} + +/// Whether `names` holds `name`, ignoring case. +fn includes_name(names: &[String], name: &str) -> bool { + names + .iter() + .any(|defined| defined.eq_ignore_ascii_case(name)) +} + +/// The relation of `sources` read as `name`, ignoring case. +fn source_named<'s>(sources: &'s [ScopeSource], name: &str) -> Option<&'s ScopeSource> { + sources + .iter() + .find(|source| source.name.eq_ignore_ascii_case(name)) +} + +/// The one statement `text` parses to, in the first dialect that parses it. +fn parsed(text: &str) -> Result { + let mut first_error: Option = None; + for dialect in PARSE_DIALECTS { + match polyglot_sql::parse(text, dialect) { + Ok(mut statements) if statements.len() == 1 => { + if let Some(statement) = statements.pop() { + return Ok(statement); + } + } + Ok(_) => { + first_error.get_or_insert_with(|| "it holds more than one statement".to_owned()); + } + Err(error) => { + first_error.get_or_insert_with(|| error.to_string()); + } + } + } + Err(first_error.unwrap_or_default()) +} + +/// The byte offset in `sql` of byte `offset` of `sql` with its references as placeholders. +fn original_offset(offset: usize, sql: &str, references: &[ParameterReference]) -> usize { + let mut moved: isize = 0; + for reference in references { + let placed_start: isize = reference.start as isize + moved; + if (offset as isize) <= placed_start { + break; + } + moved += placeholder(&reference.name).len() as isize + - (reference.end - reference.start) as isize; + } + usize::try_from(offset as isize - moved) + .unwrap_or(0) + .min(sql.len()) +} + +/// Byte offsets of a text's character offsets. +struct CharOffsets { + bytes: Option>, +} + +impl CharOffsets { + fn new(text: &str) -> Self { + if text.is_ascii() { + return Self { bytes: None }; + } + let mut bytes: Vec = text.char_indices().map(|(offset, _)| offset).collect(); + bytes.push(text.len()); + Self { bytes: Some(bytes) } + } + + fn byte(&self, character: usize) -> usize { + match &self.bytes { + Some(bytes) => bytes.get(character).copied().unwrap_or(character), + None => character, + } + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs similarity index 74% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs index c8e3ed4db7..96c8c22834 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs @@ -3,28 +3,33 @@ use std::sync::Arc; use crate::compiler::main::function_type_validation::function_type_error; -use crate::sql_macros::_helpers::arguments::{ - ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, -}; -use crate::sql_macros::_helpers::body::{expand_body, with_line_comment_closed}; -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs, outer_sql_calls}; -use crate::sql_macros::_helpers::lexing::{Token, line_column, tokens}; -use crate::sql_macros::_helpers::literal_positions::{ +use crate::sql_macros::_helpers::parsing::body::{expand_body, with_line_comment_closed}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs, outer_sql_calls}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; -use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; -use crate::sql_macros::_helpers::placement::top_level_parts; -use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::rendering::arguments::{ + ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, +}; +use crate::sql_macros::_helpers::rendering::cte_rename::{renamed_ctes, take_cte_argument}; +use crate::sql_macros::_helpers::rendering::placement::top_level_parts; +use crate::sql_macros::_helpers::rendering::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::typing::return_types::returns_type_error; +use crate::sql_macros::_helpers::typing::returns_check::{check_body_returns, check_body_types}; +use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ - ARGUMENT_CODE, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, MISPLACED_CALL_CODE, - NAME_CLASH_CODE, + ARGUMENT_CODE, CTE_ARGUMENT, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, + MISPLACED_CALL_CODE, NAME_CLASH_CODE, }; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallPosition, - CallScope, InsertedNames, ParameterKind, RenderedArgument, SqlMacroCall, SqlMacroCheck, - SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TextContext, TypedOutput, + ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallArgument, + CallPosition, CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, + RenderedDeclarations, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, + SqlMacroLocation, TextContext, TypedCall, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -123,10 +128,13 @@ fn located(failure: ExpansionFailure, location: &SqlMacroLocation) -> Expa pub(crate) fn expand_text( host: &H, context: &TextContext<'_>, -) -> Result> { +) -> Result> { let calls: Vec = sql_macro_calls(context.text)?; if calls.is_empty() { - return host.render_declarations(context.scope, context.text); + return Ok(ExpandedText { + sql: host.render_declarations(context.scope, context.text)?.sql, + calls: Vec::new(), + }); } let scanned: Vec = tokens(context.text).map_err(|_| { SqlMacroError::new( @@ -135,17 +143,23 @@ pub(crate) fn expand_text( ) })?; let mut inserted: InsertedNames = InsertedNames::default(); - let mut spliced: String = String::with_capacity(context.text.len()); + let mut spliced_text: String = String::with_capacity(context.text.len()); + let mut typed: Vec = Vec::new(); let mut cursor: usize = 0; for call in &calls { let (expansion, updated) = expand_call(host, context, (&scanned, inserted), call)?; inserted = updated; - spliced.push_str(&context.text[cursor..call.start]); - spliced.push_str(&expansion); + spliced_text.push_str(&context.text[cursor..call.start]); + typed.extend(shifted(expansion.calls, spliced_text.len())); + spliced_text.push_str(&expansion.sql); cursor = call.replaced_end; } - spliced.push_str(&context.text[cursor..]); - host.render_declarations(context.scope, &spliced) + spliced_text.push_str(&context.text[cursor..]); + let rendered: RenderedDeclarations = host.render_declarations(context.scope, &spliced_text)?; + Ok(ExpandedText { + calls: spliced(typed, &rendered.splices), + sql: rendered.sql, + }) } /// One call's expansion as it replaces the call text in `context.text`. @@ -154,7 +168,7 @@ pub(crate) fn expand_call( context: &TextContext<'_>, caller: (&[Token], InsertedNames), call: &SqlMacroCall, -) -> Result<(String, InsertedNames), ExpansionFailure> { +) -> Result<(ExpandedText, InsertedNames), ExpansionFailure> { let (scanned, inserted) = caller; let definition: Arc = host.definition(context.scope, &call.name)?; if let Some(position) = context.stack.iter().position(|name| *name == call.name) { @@ -176,8 +190,11 @@ pub(crate) fn expand_call( } check_adapter_rules(&definition, host.adapter())?; let arguments_text: &str = &context.text[call.arguments.0..call.arguments.1]; - let bound: Vec = split_arguments(arguments_text) - .and_then(|arguments| bind_arguments(&definition, &arguments)) + let (cte_name, arguments): (Option, Vec) = + split_arguments(arguments_text) + .and_then(|arguments| take_cte_argument(&definition, arguments)) + .map_err(|error| error.at(body_location(&definition)))?; + let bound: Vec = bind_arguments(&definition, &arguments) .map_err(|error| error.at(body_location(&definition)))?; let macro_scope: SqlMacroScope = SqlMacroScope::Macro(Arc::clone(&definition)); let mut rendered: Vec = Vec::with_capacity(bound.len()); @@ -196,7 +213,7 @@ pub(crate) fn expand_call( } let mut stack: Vec = context.stack.to_vec(); stack.push(call.name.clone()); - let body: String = expand_text( + let body: ExpandedText = expand_text( host, &TextContext { text: &definition.body, @@ -206,11 +223,47 @@ pub(crate) fn expand_call( }, ) .map_err(|failure| located(failure, &body_location(&definition)))?; - let expansion: BodyExpansion = expand_body(&definition, &body, &rendered) + let body: ExpandedText = match &cte_name { + Some(name) => renamed_ctes(&body, name).map_err(|reason| { + SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!( + "body of macro {}() is not a valid CTE-list macro: {reason}", + definition.name + ), + ) + .at(body_location(&definition)) + })?, + None => body, + }; + check_body_returns( + &definition, + &body, + (host.adapter(), body_location(&definition)), + )?; + check_body_types( + &definition, + &body, + (host.adapter(), body_location(&definition)), + )?; + let mut expansion: BodyExpansion = expand_body(&definition, &body, &rendered) .map_err(|error| error.at(body_location(&definition)))?; + expansion.like = definition.returns_like.as_deref().and_then(|name| { + rendered + .iter() + .find(|argument| { + definition.parameters[argument.parameter] + .name + .eq_ignore_ascii_case(name) + }) + .map(|argument| argument.sql.clone()) + }); if call.in_python { return Ok(( - python_string(&with_line_comment_closed(&expansion.sql, true)), + ExpandedText { + sql: python_string(&with_line_comment_closed(&expansion.sql, true)), + calls: Vec::new(), + }, inserted, )); } @@ -221,26 +274,14 @@ pub(crate) fn expand_call( call.replaced_end, ); if matches!(context.scope, SqlMacroScope::Caller) && context.stack.is_empty() { - let output: Option = match (definition.form, &definition.returns) { - (SqlMacroForm::Expression, Some(returns)) => site - .whole_item_alias() - .zip(call_branch(context.text, scanned, call.start)) - .map(|(name, branch)| TypedOutput { - name, - returns: returns.clone(), - branch, - }), - _ => None, - }; let check: SqlMacroCheck = SqlMacroCheck { macro_name: definition.name.clone(), scope: call_scope(context.text, scanned, call.start), plain_select: plain_select(context.text, scanned, call.start), arguments: argument_checks(&definition, &bound, &rendered), call: None, - output, }; - if !check.arguments.is_empty() || check.output.is_some() { + if !check.arguments.is_empty() { host.record_check(check, (call.start, call.end)); } } @@ -296,26 +337,36 @@ fn rendered_argument( .at(parameter_location.clone()) .into() }; - let sql: String = match (¶meter.kind, &shape) { - (ParameterKind::Collection { .. }, ArgumentShape::InlineCollection) => { - host.render_collection(text)? - } + let rendered: ExpandedText = match (¶meter.kind, &shape) { + (ParameterKind::Collection { .. }, ArgumentShape::InlineCollection) => ExpandedText { + sql: host.render_collection(text)?, + calls: Vec::new(), + }, (ParameterKind::Collection { .. }, ArgumentShape::ConstantReference { name }) => { if host.constant_shape(scope, name)? != ConstantShape::Collection { return Err(expects("a list or set constant")); } - host.render_declarations(scope, text)? + ExpandedText { + sql: host.render_declarations(scope, text)?.sql, + calls: Vec::new(), + } } ( - ParameterKind::Constant { .. } | ParameterKind::Any, + ParameterKind::Constant { .. } | ParameterKind::Any { .. }, ArgumentShape::ConstantReference { name }, ) => { if host.constant_shape(scope, name)? != ConstantShape::Scalar { return Err(expects("a scalar constant")); } - host.render_declarations(scope, text)? + ExpandedText { + sql: host.render_declarations(scope, text)?.sql, + calls: Vec::new(), + } } - (ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. }, _) => text.to_owned(), + (ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. }, _) => ExpandedText { + sql: text.to_owned(), + calls: Vec::new(), + }, _ => expand_text( host, &TextContext { @@ -328,8 +379,9 @@ fn rendered_argument( }; Ok(RenderedArgument { parameter: argument.parameter, - sql, + sql: rendered.sql, shape, + calls: rendered.calls, }) } @@ -351,13 +403,40 @@ fn python_string(sql: &str) -> String { quoted } -/// The expansion placed at the call, or why it cannot stand there or clashes. +/// The expansion placed at the call with the calls it holds, or why it cannot stand there. fn placed( definition: &SqlMacroDefinition, (site, inserted, line_continues): (&CallSite<'_>, InsertedNames, bool), expansion: &BodyExpansion, owner: &str, -) -> Result<(String, InsertedNames), SqlMacroError> { +) -> Result<(ExpandedText, InsertedNames), SqlMacroError> { + let (text, inserted, prefix) = placed_text( + definition, + (site, inserted, line_continues), + expansion, + owner, + )?; + let mut calls: Vec = shifted(expansion.calls.clone(), prefix).collect(); + if definition.form == SqlMacroForm::Expression { + let comma: usize = site.expression_separator().len(); + calls.push(TypedCall { + start: 0, + end: text.len() - comma, + macro_name: definition.name.clone(), + returns: definition.returns.clone(), + returns_like: expansion.like.clone(), + }); + } + Ok((ExpandedText { sql: text, calls }, inserted)) +} + +/// The expansion's text placed at the call, and where the expansion starts in it. +fn placed_text( + definition: &SqlMacroDefinition, + (site, inserted, line_continues): (&CallSite<'_>, InsertedNames, bool), + expansion: &BodyExpansion, + owner: &str, +) -> Result<(String, InsertedNames, usize), SqlMacroError> { let closed = |after: &str| { let followed: bool = match after.chars().next() { None => line_continues, @@ -375,9 +454,9 @@ fn placed( } let comma: &str = site.expression_separator(); if site.expression_needs_parentheses() && !is_atomic(&expansion.sql) { - Ok((format!("({}){comma}", closed(")")), inserted)) + Ok((format!("({}){comma}", closed(")")), inserted, 1)) } else { - Ok((format!("{}{comma}", closed(comma)), inserted)) + Ok((format!("{}{comma}", closed(comma)), inserted, 0)) } } SqlMacroForm::CteList => { @@ -399,7 +478,11 @@ fn placed( ("CTE", owner), )?; let (before, after) = site.cte_separators(); - Ok((format!("{before}{}{after}", closed(after)), inserted)) + Ok(( + format!("{before}{}{after}", closed(after)), + inserted, + before.len(), + )) } SqlMacroForm::ColumnList => { if position != CallPosition::SelectList { @@ -429,7 +512,11 @@ fn placed( ("column", owner), )?; let (before, after) = site.column_separators(); - Ok((format!("{before}{}{after}", closed(after)), inserted)) + Ok(( + format!("{before}{}{after}", closed(after)), + inserted, + before.len(), + )) } } } @@ -445,6 +532,12 @@ fn check_clashes( let clash_location: SqlMacroLocation = body_offset_location(definition, definition.body.find(name.as_str()).unwrap_or(0)); let lists_with = |plural: &str| { + if definition.form == SqlMacroForm::CteList { + return format!( + "pass {CTE_ARGUMENT} := to give the CTEs one call inserts their own names, as in @sql.{}(..., {CTE_ARGUMENT} := first_orders), or rename one of the {plural}", + definition.name + ); + } format!( "rename one of the {plural}; a model can't call a {} macro whose {noun} names it already uses", definition.form.label() @@ -531,9 +624,9 @@ fn check_adapter_types( .iter() .map(|(_, data_type)| data_type.as_str()) .collect(), - ParameterKind::Any | ParameterKind::Enum { .. } | ParameterKind::ColumnOf { .. } => { - Vec::new() - } + ParameterKind::Any { .. } + | ParameterKind::Enum { .. } + | ParameterKind::ColumnOf { .. } => Vec::new(), }; for data_type in types { if let Some(error) = invalid( @@ -545,9 +638,15 @@ fn check_adapter_types( } } if let Some(returns) = &definition.returns - && let Some(error) = invalid(returns, format!("returns of macro {}()", definition.name)) + && let Some(message) = returns_type_error( + returns, + adapter, + &format!("returns of macro {}()", definition.name), + ) { - return Err(error.at(macro_location(definition, 1, 1))); + return Err(SqlMacroError::new(INVALID_DEFINITION_CODE, message) + .helped("use a SQL type the warehouse accepts, as in schema declarations") + .at(macro_location(definition, 1, 1))); } Ok(()) } @@ -583,6 +682,14 @@ fn argument_checks( data_type: data_type.clone(), } } + ( + ParameterKind::Any { + family: Some(family), + }, + ArgumentShape::Expression | ArgumentShape::Identifier { .. }, + ) if !rendered.sql.contains('@') && !rendered.sql.contains("__") => { + ArgumentExpectation::Family { family: *family } + } _ => continue, }; checks.push(ArgumentCheck { @@ -603,11 +710,6 @@ fn call_scope(text: &str, scanned: &[Token], offset: usize) -> CallScope { } } -/// The branch of its scope's set operation the call at byte `offset` of `text` stands in. -fn call_branch(text: &str, scanned: &[Token], offset: usize) -> Option { - top_level_parts(text, scanned)?.branch_of(text, scanned, offset) -} - /// Whether the call at byte `offset` of `text` stands in its scope's one select. fn plain_select(text: &str, scanned: &[Token], offset: usize) -> bool { top_level_parts(text, scanned).is_some_and(|parts| parts.plain_select(text, scanned, offset)) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs new file mode 100644 index 0000000000..9b07958fe7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod arguments; +pub(crate) mod cte_rename; +pub(crate) mod expansion; +pub(crate) mod placement; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs similarity index 81% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs index 9991907baf..d365663e92 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs @@ -1,8 +1,8 @@ //! Where a call stands in its caller: its query part, separators and the caller's names. -use crate::sql_macros::_helpers::header::unquoted; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token}; -use crate::sql_macros::constants::{AS_KEYWORD, AT_SIGN, COMMA, SELECT_KEYWORD, WITH_KEYWORD}; +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token}; +use crate::sql_macros::constants::{AT_SIGN, COMMA, SELECT_KEYWORD, WITH_KEYWORD}; use crate::sql_macros::models::{CallPosition, CallScope}; /// Tokens before the `(` of `DISTINCT ON (...)`. @@ -11,10 +11,6 @@ const DISTINCT_ON_TOKENS: usize = 2; const QUERY_OPENERS: [&str; 4] = ["SELECT", "WITH", "VALUES", "FROM"]; /// Keywords that combine the selects of a query. const SET_OPERATIONS: [&str; 3] = ["UNION", "INTERSECT", "EXCEPT"]; -/// A star, which an `EXCEPT` column list may follow as a modifier, not a set operation. -const STAR: &str = "*"; -/// The words that may stand between a set operation and its parenthesised right branch. -const SET_OPERATION_MODIFIERS: [&str; 3] = ["ALL", "DISTINCT", "NAME"]; /// Tokens from `@` to a macro call's name: `@`, the kind, `.` and the name. const CALL_PREFIX_TOKENS: usize = 4; /// Keywords that open the clause a call stands in, read backwards from the call. @@ -256,48 +252,6 @@ impl<'a> CallSite<'a> { } } - /// The alias of the select-list item the call is the whole of, as `@sql.m(x) AS name`. - pub(crate) fn whole_item_alias(&self) -> Option { - let opens_item: bool = self.previous_token().is_some_and(|token| { - matches!( - token.text(self.sql).to_ascii_uppercase().as_str(), - "SELECT" | "DISTINCT" | "ALL" | "," - ) - }); - if !opens_item || self.position() != CallPosition::SelectList { - return None; - } - let next: usize = self.next?; - let alias_index: usize = if self.tokens[next].is_word(self.sql, AS_KEYWORD) { - next + 1 - } else { - next - }; - let alias: &Token = self.tokens.get(alias_index)?; - let alias_text: String = alias.text(self.sql).to_ascii_uppercase(); - let named: bool = match alias.kind { - TokenKind::Quoted => true, - TokenKind::Word => { - !SELECT_LIST_ENDS.contains(&alias_text.as_str()) - && !EXPRESSION_OPERATOR_WORDS.contains(&alias_text.as_str()) - && alias_text != AS_KEYWORD - } - _ => false, - }; - if !named { - return None; - } - let ends_item: bool = match self.tokens.get(alias_index + 1) { - None => true, - Some(after) => { - let text: String = after.text(self.sql).to_ascii_uppercase(); - matches!(text.as_str(), "," | ")" | ";") - || (after.kind == TokenKind::Word && SELECT_LIST_ENDS.contains(&text.as_str())) - } - }; - ends_item.then(|| unquoted(alias.text(self.sql))) - } - /// Whether the token after the call ends a select-list item. fn ends_select_item(&self) -> bool { let Some(next) = self.next_token() else { @@ -435,7 +389,7 @@ pub(crate) fn cte_names(sql: &str, tokens: &[Token], with_index: usize) -> Vec<( if !matches!(token.kind, TokenKind::Word | TokenKind::Quoted) { break; } - let name: String = crate::sql_macros::_helpers::header::unquoted(token.text(sql)); + let name: String = crate::sql_macros::_helpers::parsing::header::unquoted(token.text(sql)); let mut cursor: usize = index + 1; if tokens.get(cursor).is_some_and(|after| after.is(sql, "(")) { match closing_token(sql, tokens, cursor) { @@ -534,7 +488,7 @@ fn item_name(sql: &str, item: &[Token]) -> Option<(String, usize)> { }; named.then(|| { ( - crate::sql_macros::_helpers::header::unquoted(last.text(sql)), + crate::sql_macros::_helpers::parsing::header::unquoted(last.text(sql)), last.start, ) }) @@ -615,74 +569,6 @@ impl QueryParts { } } -impl QueryParts { - /// The set-operation branch byte `offset` stands in; `None` where it cannot be counted. - pub(crate) fn branch_of(&self, text: &str, scanned: &[Token], offset: usize) -> Option { - let range: (usize, usize) = match self.scope_of(offset) { - CallScope::WithList => return None, - CallScope::Cte(name) => self.ctes.iter().find(|cte| cte.name == name)?.body, - CallScope::Main => (self.main, text.len()), - }; - let inside: Vec<(usize, &Token)> = scanned - .iter() - .enumerate() - .filter(|(_, token)| range.0 <= token.start && token.end <= range.1) - .collect(); - if inside - .first() - .is_some_and(|(_, token)| token.is_word(text, WITH_KEYWORD)) - { - return None; - } - let mut depth: usize = 0; - let mut branch: usize = 0; - let mut at_call: Option = None; - let mut previous_at_top: Option<&Token> = None; - for (index, token) in inside { - if at_call.is_none() && token.start >= offset { - if depth != 0 { - return None; - } - at_call = Some(branch); - } - if token.kind == TokenKind::Punct { - match token.text(text) { - "(" | "[" | "{" => { - let query: bool = scanned - .get(index + 1) - .is_some_and(|next| opens_query(text, next)); - let branch_start: bool = previous_at_top.is_none_or(|previous| { - let word: &str = previous.text(text); - is_set_operation(word) - || SET_OPERATION_MODIFIERS - .iter() - .any(|modifier| word.eq_ignore_ascii_case(modifier)) - }); - if depth == 0 && query && branch_start { - return None; - } - depth += 1; - } - ")" | "]" | "}" => depth = depth.saturating_sub(1), - _ => {} - } - } - if depth == 0 { - let star_modifier: bool = - previous_at_top.is_some_and(|previous| previous.text(text) == STAR); - if token.kind == TokenKind::Word - && is_set_operation(token.text(text)) - && !star_modifier - { - branch += 1; - } - previous_at_top = Some(token); - } - } - at_call - } -} - /// Whether `token` opens a query, as the first token inside a parenthesis. fn opens_query(text: &str, token: &Token) -> bool { QUERY_OPENERS diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs new file mode 100644 index 0000000000..0b5e9f1273 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod probes; +pub(crate) mod return_types; +pub(crate) mod returns_check; +pub(crate) mod typed_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs similarity index 91% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs index 472251f765..583231d189 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs @@ -4,10 +4,10 @@ use polyglot_sql::ValidationSchema; use serde_json::{Value, json}; use crate::query_analysis::main::output_types::output_types; -use crate::sql_macros::_helpers::arguments::{fits, type_family}; -use crate::sql_macros::_helpers::header::unquoted; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, tokens}; -use crate::sql_macros::_helpers::placement::{QueryParts, top_level_parts}; +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::_helpers::rendering::arguments::{fits, type_family}; +use crate::sql_macros::_helpers::rendering::placement::{QueryParts, top_level_parts}; use crate::sql_macros::constants::{ ALIAS_PRECEDING_KEYWORDS, FROM_KEYWORD, INFERRED_TEXT_TYPE, PROBE_COLUMN_PREFIX, PROBE_TAIL_ENDS, STAR_PROJECTION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, @@ -193,7 +193,7 @@ pub(crate) fn probe_plan(sql: &str, checks: &[SqlMacroCheck]) -> ProbePlan { evidence[index] = Some(Evidence::Columns(plan.probes.len() - 1)); } } - ArgumentExpectation::Typed { .. } => { + ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let group: usize = match typed_probe_index(&scopes, &check.scope) { Some(group) => group, None => { @@ -345,7 +345,7 @@ pub(crate) fn check_findings( )); } } - ArgumentExpectation::Typed { data_type } => { + ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let Some(Some(Evidence::Column(probe, column))) = evidence.get(index) else { continue; }; @@ -355,6 +355,28 @@ pub(crate) fn check_findings( let Some(Some(actual)) = column_type(found, column) else { continue; }; + let data_type: &str = match &argument.expected { + ArgumentExpectation::Family { family } => { + if !family.admits(type_family(actual, dialect)) { + findings.push(finding( + argument, + format!( + "argument {parameter} of macro {name}() expects {}; {} is {}", + family.noun(), + argument.text, + shown_type(actual) + ), + Some(format!( + "pass {} or declare {parameter} with a wider type", + family.noun() + )), + )); + } + continue; + } + ArgumentExpectation::Typed { data_type } => data_type, + _ => continue, + }; if !compatible(Some(actual), data_type, dialect) { findings.push(finding( argument, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs new file mode 100644 index 0000000000..469ac054ee --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs @@ -0,0 +1,55 @@ +//! A SQL macro's declared `returns` type checked against the adapter's SQL analysis dialect. + +use polyglot_sql::expressions::DataType; +use polyglot_sql::{Dialect, DialectType, Expression}; + +use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; + +/// Why `returns` is not a type the adapter's SQL analysis dialect parses and can hold. +pub(crate) fn returns_type_error(returns: &str, adapter: &str, context: &str) -> Option { + let dialect_name: &str = adapter_dialect(adapter); + let Ok(dialect) = dialect_name.parse::() else { + return None; + }; + let reason: String = match polyglot_sql::parse_data_type(returns, dialect) { + Ok(parsed) => decimal_error(&parsed, dialect)?, + Err(error) => error.to_string(), + }; + Some(format!( + "{context} type '{returns}' is not valid for adapter '{adapter}' SQL analysis dialect \ + '{dialect_name}': {reason}" + )) +} + +/// Why a decimal within `data_type` has a precision or scale `dialect` cannot hold. +fn decimal_error(data_type: &DataType, dialect: DialectType) -> Option { + match data_type { + DataType::Decimal { + precision: Some(precision), + scale, + } => { + if let Some(scale) = scale.filter(|scale| scale > precision) { + return Some(format!( + "DECIMAL scale {scale} is larger than its precision {precision}" + )); + } + match Dialect::get(dialect).generate(&Expression::DataType(data_type.clone())) { + Ok(_) => None, + Err(polyglot_sql::error::Error::Unsupported { feature, .. }) => Some(feature), + Err(_) => None, + } + } + DataType::Array { element_type, .. } | DataType::List { element_type } => { + decimal_error(element_type, dialect) + } + DataType::Nullable { inner } => decimal_error(inner, dialect), + DataType::Map { + key_type, + value_type, + } => decimal_error(key_type, dialect).or_else(|| decimal_error(value_type, dialect)), + DataType::Struct { fields, .. } => fields + .iter() + .find_map(|field| decimal_error(&field.data_type, dialect)), + _ => None, + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs new file mode 100644 index 0000000000..0f8e0d56e2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -0,0 +1,322 @@ +//! An expression macro's body typed from its declared arguments and checked against `returns`. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use polyglot_sql::expressions::DataType; +use polyglot_sql::{DialectType, SchemaValidationOptions, ValidationSchema, validate_with_schema}; +use serde_json::{Value, json}; + +use crate::query_analysis::main::annotated_output_types::annotated_output_types; +use crate::sql_macros::_helpers::parsing::body::expand_body; +use crate::sql_macros::_helpers::rendering::arguments::type_family; +use crate::sql_macros::_helpers::typing::typed_calls::{ + markable_calls, shifted, typed_sql, without_marker_calls, +}; +use crate::sql_macros::constants::{ + BODY_TYPE_CODE, INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, TYPE_ERROR_CODES, + TYPE_WARNING_CODES, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, +}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, BodyType, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, + SqlMacroForm, SqlMacroLocation, TypeFamily, TypedCall, +}; +use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::TypeNormalization; + +/// Each body's type, by the probe that typed it, so a macro is typed once per process. +static BODY_TYPES: OnceLock>> = OnceLock::new(); +/// Each body's first type error, by the probe that found it. +static TYPE_ERRORS: OnceLock>>> = OnceLock::new(); + +/// The dialect an adapter's SQL is analysed in. +pub(crate) fn adapter_dialect(adapter: &str) -> &str { + match adapter { + "motherduck" => "duckdb", + "" => "generic", + other => other, + } +} + +/// A parameter's stand-in column type, with `ANY` families widened only when `families` is set. +fn stand_in_type(kind: &ParameterKind, families: bool) -> Option<&str> { + match kind { + ParameterKind::Expression { data_type } | ParameterKind::Constant { data_type } => { + Some(data_type) + } + ParameterKind::Collection { data_type, .. } => Some(data_type), + ParameterKind::Any { + family: Some(family), + } if families => Some(family.widest()), + ParameterKind::Any { .. } + | ParameterKind::Enum { .. } + | ParameterKind::Relation { .. } + | ParameterKind::ColumnOf { .. } => None, + } +} + +/// The name of parameter `index`'s stand-in column. +fn stand_in_column(index: usize) -> String { + format!("p{index}") +} + +/// The body selected over stand-in arguments, and their relation, unless it reads relations. +fn body_probe( + definition: &SqlMacroDefinition, + body: &ExpandedText, + (families, dialect): (bool, &str), +) -> Option<(String, Value)> { + let mut arguments: Vec = Vec::with_capacity(definition.parameters.len()); + let mut columns: Vec = Vec::with_capacity(definition.parameters.len()); + for (index, parameter) in definition.parameters.iter().enumerate() { + if matches!( + parameter.kind, + ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. } + ) { + return None; + } + let column: String = stand_in_column(index); + let sql: String = match parameter.kind { + ParameterKind::Collection { .. } => format!("({STAND_IN_RELATION}.{column})"), + _ => format!("{STAND_IN_RELATION}.{column}"), + }; + columns.push(json!({ + "name": column, + "type": stand_in_type(¶meter.kind, families).unwrap_or(UNKNOWN_TYPE_HINT), + })); + arguments.push(RenderedArgument { + parameter: index, + sql, + shape: ArgumentShape::Expression, + calls: Vec::new(), + }); + } + let expansion = match expand_body(definition, body, &arguments) { + Ok(expansion) => expansion, + Err(_) => return None, + }; + let prefix: &str = "SELECT ("; + let unmarked: String = format!("{prefix}{}) AS b FROM {STAND_IN_RELATION}", expansion.sql); + let calls: Vec = shifted(expansion.calls, prefix.len()).collect(); + let (probe, _) = typed_sql(&unmarked, &markable_calls(&unmarked, &calls, dialect)); + let table: Value = json!({"name": STAND_IN_RELATION, "columns": columns}); + Some((probe, table)) +} + +/// What inference knows of the body's type. +fn inferred_body_type(probe: &str, table: Value, dialect: &str) -> BodyType { + let Ok(schema) = serde_json::from_value::(json!({ "tables": [table] })) + else { + return BodyType::Unknown; + }; + let Some(outputs) = annotated_output_types(probe, dialect, &schema) else { + return BodyType::Unknown; + }; + let Some((_, shown, annotated)) = outputs.into_iter().next() else { + return BodyType::Unknown; + }; + match (shown, annotated) { + (Some(shown), _) if !shown.to_ascii_uppercase().contains(UNKNOWN_TYPE_HINT) => { + BodyType::Known(shown) + } + (Some(shown), Some(data_type)) + if data_type.contains_unknown() && data_type != DataType::Unknown => + { + BodyType::PartlyKnown { shown, data_type } + } + _ => BodyType::Unknown, + } +} + +/// Whether `returns` cannot fill the unknown components of `partial` without changing its shape. +fn changes_shape(returns: &str, partial: &DataType, dialect: &str) -> bool { + let Ok(dialect_type) = dialect.parse::() else { + return false; + }; + match polyglot_sql::parse_data_type(returns, dialect_type) { + Ok(declared) => !partial.is_refined_by(&declared), + Err(_) => false, + } +} + +/// Whether body type `inferred` is outside the family of `returns`, or another non-family type. +fn disagrees(returns: &str, inferred: &str, adapter: &str) -> bool { + let (declared, found) = ( + type_family(returns, adapter), + type_family(inferred, adapter), + ); + if declared != TypeFamily::Unknown && found != TypeFamily::Unknown { + return declared != found; + } + let dialect: &str = adapter_dialect(adapter); + match ( + normalize_type(returns, dialect), + normalize_type(inferred, dialect), + ) { + ( + Ok(TypeNormalization { + normalized: declared, + .. + }), + Ok(TypeNormalization { + normalized: found, .. + }), + ) => declared != found, + _ => false, + } +} + +/// The verdict `cache` holds under `key`, or `compute`'s verdict, cached for the process. +fn remembered( + cache: &OnceLock>>, + key: String, + compute: impl FnOnce() -> T, +) -> T { + let verdicts = cache.get_or_init(|| Mutex::new(HashMap::new())); + let mut verdicts = match verdicts.lock() { + Ok(verdicts) => verdicts, + Err(poisoned) => poisoned.into_inner(), + }; + if let Some(known) = verdicts.get(&key) { + return known.clone(); + } + let verdict: T = compute(); + verdicts.insert(key, verdict.clone()); + verdict +} + +/// Rejects an expression macro whose body's known type is not its declared `returns`. +pub(crate) fn check_body_returns( + definition: &SqlMacroDefinition, + body: &ExpandedText, + (adapter, location): (&str, SqlMacroLocation), +) -> Result<(), SqlMacroError> { + if definition.form != SqlMacroForm::Expression { + return Ok(()); + } + let Some(returns) = definition.returns.as_deref() else { + return Ok(()); + }; + let Some((probe, table)) = body_probe(definition, body, (false, adapter_dialect(adapter))) + else { + return Ok(()); + }; + let dialect: &str = adapter_dialect(adapter); + let key: String = format!("returns\n{dialect}\n{table}\n{probe}"); + let body_type: BodyType = remembered(&BODY_TYPES, key, || { + inferred_body_type(&probe, table, dialect) + }); + let partial: bool = matches!(body_type, BodyType::PartlyKnown { .. }); + let (shown, wrong): (String, bool) = match body_type { + BodyType::Unknown => return Ok(()), + BodyType::Known(inferred) if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) => ( + VARCHAR_TYPE.to_owned(), + disagrees(returns, VARCHAR_TYPE, adapter), + ), + BodyType::Known(inferred) => { + let wrong: bool = disagrees(returns, &inferred, adapter); + (inferred, wrong) + } + BodyType::PartlyKnown { shown, data_type } => { + let wrong: bool = changes_shape(returns, &data_type, dialect); + (shown, wrong) + } + }; + if !wrong { + return Ok(()); + } + Err(SqlMacroError::new( + RETURNS_MISMATCH_CODE, + format!( + "macro {}() declares returns {returns}, but its body is {shown}", + definition.name + ), + ) + .at(location) + .helped(if partial { + format!( + "declare a returns of the body's shape {shown} with its unknown parts filled in, CAST the body to {returns}, or remove returns to use the inferred type" + ) + } else { + format!( + "declare returns {shown}, CAST the body to {returns}, or remove returns to use the inferred type" + ) + })) +} + +/// The first type error the body's probe shows, as polyglot reports it. +fn body_type_error(probe: &str, table: Value, dialect: &str) -> Option { + let Ok(schema) = serde_json::from_value::(json!({ "tables": [table] })) + else { + return None; + }; + let dialect_type: DialectType = match dialect.parse() { + Ok(dialect_type) => dialect_type, + Err(_) => return None, + }; + let options: SchemaValidationOptions = SchemaValidationOptions { + check_types: true, + strict: Some(false), + ..Default::default() + }; + validate_with_schema(probe, dialect_type, &schema, &options) + .errors + .into_iter() + .find(|error| is_type_code(&error.code)) + .map(|error| without_marker_calls(&error.message).into_owned()) +} + +/// Whether a polyglot validation code reports an operand or argument of the wrong type. +fn is_type_code(code: &str) -> bool { + let number: usize = match code.get(1..).map(str::parse::) { + Some(Ok(number)) => number, + _ => return false, + }; + (code.starts_with('E') && (TYPE_ERROR_CODES.0..=TYPE_ERROR_CODES.1).contains(&number)) + || (code.starts_with('W') + && (TYPE_WARNING_CODES.0..=TYPE_WARNING_CODES.1).contains(&number)) +} + +/// Rejects a body that misuses an `ANY` family argument, typed as the family's widest member. +pub(crate) fn check_body_types( + definition: &SqlMacroDefinition, + body: &ExpandedText, + (adapter, location): (&str, SqlMacroLocation), +) -> Result<(), SqlMacroError> { + let families: Vec = definition + .parameters + .iter() + .filter_map(|parameter| match parameter.kind { + ParameterKind::Any { + family: Some(family), + } => Some(format!("{} ANY {}", parameter.name, family.keyword())), + _ => None, + }) + .collect(); + if families.is_empty() { + return Ok(()); + } + let Some((probe, table)) = body_probe(definition, body, (true, adapter_dialect(adapter))) + else { + return Ok(()); + }; + let dialect: &str = adapter_dialect(adapter); + let key: String = format!("types\n{dialect}\n{table}\n{probe}"); + let found: Option = remembered(&TYPE_ERRORS, key, || { + body_type_error(&probe, table, dialect) + }); + let Some(found) = found else { + return Ok(()); + }; + Err(SqlMacroError::new( + BODY_TYPE_CODE, + format!( + "body of macro {}() does not accept every value of {}: {found}", + definition.name, + families.join(", ") + ), + ) + .at(location) + .helped("an ANY family argument may be any type of its family; the body is checked with the family's widest type, so CAST the argument or narrow its declared type")) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs new file mode 100644 index 0000000000..f90abe1f48 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -0,0 +1,296 @@ +//! Expression macro calls followed through the edits that move them, and their typed markers. + +use std::borrow::Cow; + +use polyglot_sql::expressions::{TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION}; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; + +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; +use crate::sql_macros::constants::{MARKER_PARSE_DIALECTS, NULL_KEYWORD}; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` moved `offset` bytes later. +pub(crate) fn shifted(calls: Vec, offset: usize) -> impl Iterator { + calls.into_iter().map(move |call| TypedCall { + start: call.start + offset, + end: call.end + offset, + ..call + }) +} + +/// Where byte `position` of the source lands, or `None` inside a replaced range. +fn spliced_position(position: usize, splices: &[TextSplice]) -> Option { + let mut moved: isize = 0; + for splice in splices { + let (start, end) = splice.source; + if end <= position { + moved += (splice.output.1 - splice.output.0) as isize - (end - start) as isize; + } else if start < position { + return None; + } else { + break; + } + } + let Ok(moved_position) = usize::try_from(position as isize + moved) else { + return None; + }; + Some(moved_position) +} + +/// `calls` through `splices`, sorted and non-overlapping; a call a splice cuts into is dropped. +pub(crate) fn spliced(calls: Vec, splices: &[TextSplice]) -> Vec { + if splices.is_empty() { + return calls; + } + calls + .into_iter() + .filter_map(|call| { + let start: usize = spliced_position(call.start, splices)?; + let end: usize = spliced_position(call.end, splices)?; + Some(TypedCall { start, end, ..call }) + }) + .collect() +} + +/// Byte offset of each character offset of `text`, one past the end included. +fn byte_offsets(text: &str) -> Option> { + if text.is_ascii() { + return None; + } + let mut offsets: Vec = text.char_indices().map(|(offset, _)| offset).collect(); + offsets.push(text.len()); + Some(offsets) +} + +/// Character-offset `spans` from `source` to `output` as byte splices. +pub(crate) fn byte_splices(source: &str, output: &str, spans: &[SplicedSpan]) -> Vec { + let (source_offsets, output_offsets) = (byte_offsets(source), byte_offsets(output)); + let byte = |offsets: &Option>, position: usize| match offsets { + Some(offsets) => offsets.get(position).copied().unwrap_or(position), + None => position, + }; + spans + .iter() + .map(|span| TextSplice { + source: ( + byte(&source_offsets, span.source_start), + byte(&source_offsets, span.source_end), + ), + output: ( + byte(&output_offsets, span.output_start), + byte(&output_offsets, span.output_end), + ), + }) + .collect() +} + +/// One marker edit: the text inserted at a byte position, ordered among edits there. +struct MarkerEdit { + position: usize, + /// Closing edits come before opening ones at one position. + opening: bool, + /// Among openings, the outermost call first; among closings, the innermost first. + order: (usize, usize), + text: String, +} + +/// `returns` as a SQL string literal. +fn quoted(returns: &str) -> String { + format!("'{}'", returns.replace('\'', "''")) +} + +/// `sql` with each call that declares `returns` wrapped in the typed marker, and each insertion. +pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { + let mut edits: Vec = Vec::new(); + for (index, call) in calls.iter().enumerate() { + let (function, asserted): (&str, String) = match (&call.returns, &call.returns_like) { + (Some(returns), _) => (TYPED_EXPRESSION_FUNCTION, quoted(returns)), + (None, Some(like)) => ( + TYPED_LIKE_EXPRESSION_FUNCTION, + with_line_comment_closed(like, true), + ), + (None, None) => continue, + }; + let valid = |position: usize| position <= sql.len() && sql.is_char_boundary(position); + if call.start >= call.end || !valid(call.start) || !valid(call.end) { + continue; + } + let inner: &str = &sql[call.start..call.end]; + if is_bare_word(inner) { + continue; + } + let line_break: &str = if with_line_comment_closed(inner, true).len() > inner.len() { + "\n" + } else { + "" + }; + edits.push(MarkerEdit { + position: call.start, + opening: true, + order: (usize::MAX - call.end, usize::MAX - index), + text: format!("{function}("), + }); + edits.push(MarkerEdit { + position: call.end, + opening: false, + order: (usize::MAX - call.start, index), + text: format!("{line_break}, {asserted})"), + }); + } + if edits.is_empty() { + return (sql.to_owned(), Vec::new()); + } + edits.sort_by_key(|edit| (edit.position, edit.opening, edit.order)); + let mut typed: String = String::with_capacity(sql.len() + edits.len() * 16); + let mut insertions: Vec = Vec::with_capacity(edits.len()); + let mut cursor: usize = 0; + for edit in edits { + typed.push_str(&sql[cursor..edit.position]); + cursor = edit.position; + insertions.push(TextSplice { + source: (edit.position, edit.position), + output: (typed.len(), typed.len() + edit.text.len()), + }); + typed.push_str(&edit.text); + } + typed.push_str(&sql[cursor..]); + (typed, insertions) +} + +/// `calls` without any whose marker would change how `sql` parses in `dialect`. +pub(crate) fn markable_calls<'c>( + sql: &str, + calls: &'c [TypedCall], + dialect: &str, +) -> Cow<'c, [TypedCall]> { + let marked: usize = calls.iter().filter(|call| is_marked(sql, call)).count(); + if marked == 0 { + return Cow::Borrowed(calls); + } + let parsed_dialect: Option = match dialect.parse::() { + Ok(parsed) => Some(parsed), + Err(_) => MARKER_PARSE_DIALECTS + .into_iter() + .find(|candidate| lowered_markers(sql, *candidate).is_some()), + }; + let Some(parse_dialect) = parsed_dialect else { + return Cow::Owned(Vec::new()); + }; + if lowered_markers(&typed_sql(sql, calls).0, parse_dialect) == Some(marked) { + return Cow::Borrowed(calls); + } + let mut kept: Vec = Vec::with_capacity(calls.len()); + let mut kept_marked: usize = 0; + for call in calls { + if !is_marked(sql, call) { + kept.push(call.clone()); + continue; + } + kept.push(call.clone()); + if lowered_markers(&typed_sql(sql, &kept).0, parse_dialect) == Some(kept_marked + 1) { + kept_marked += 1; + } else { + kept.pop(); + } + } + Cow::Owned(kept) +} + +/// Whether `typed_sql` wraps `call` in a marker. +fn is_marked(sql: &str, call: &TypedCall) -> bool { + (call.returns.is_some() || call.returns_like.is_some()) + && sql + .get(call.start..call.end) + .is_some_and(|text| !is_bare_word(text)) +} + +/// Whether `text` is one identifier or keyword other than `NULL`, which a marker could misplace. +fn is_bare_word(text: &str) -> bool { + let word: &str = text.trim(); + if word.eq_ignore_ascii_case(NULL_KEYWORD) { + return false; + } + let mut characters = word.chars(); + characters + .next() + .is_some_and(|first| first.is_alphabetic() || first == '_') + && characters.all(|character| character.is_alphanumeric() || character == '_') +} + +/// How many typed markers `sql` lowers to expression nodes in `dialect`, if it parses. +fn lowered_markers(sql: &str, dialect: DialectType) -> Option { + let statements: Vec = match polyglot_sql::parse(sql, dialect) { + Ok(statements) => statements, + Err(_) => return None, + }; + let mut lowered: usize = 0; + for statement in &statements { + lowered += statement + .dfs() + .filter(|node| is_lowered_marker(node)) + .count(); + } + Some(lowered) +} + +/// Whether `node` is a typed marker lowered to its expression node. +fn is_lowered_marker(node: &Expression) -> bool { + matches!(node, Expression::Annotated(annotated) + if annotated.type_assertion.is_some() || annotated.type_like.is_some()) +} + +/// `text` with each typed marker call, as written in a message, replaced by its first argument. +pub(crate) fn without_marker_calls(text: &str) -> Cow<'_, str> { + let lowered: String = text.to_ascii_lowercase(); + if !lowered.contains(TYPED_EXPRESSION_FUNCTION) { + return Cow::Borrowed(text); + } + let mut current: String = text.to_owned(); + while let Some(start) = current.to_ascii_lowercase().find(TYPED_EXPRESSION_FUNCTION) { + let replaced: Option = marker_argument(¤t, start) + .map(|(argument, end)| format!("{}{argument}{}", ¤t[..start], ¤t[end..])); + current = match replaced { + Some(replaced) => replaced, + None => format!( + "{}expression{}", + ¤t[..start], + ¤t[start + TYPED_EXPRESSION_FUNCTION.len()..] + ), + }; + } + Cow::Owned(current) +} + +/// The first argument of the marker call at byte `start` of `text`, and the byte after the call. +fn marker_argument(text: &str, start: usize) -> Option<(String, usize)> { + let open: usize = start + text[start..].find('(')?; + let name: &str = &text[start..open]; + if !name.eq_ignore_ascii_case(TYPED_EXPRESSION_FUNCTION) + && !name.eq_ignore_ascii_case(TYPED_LIKE_EXPRESSION_FUNCTION) + { + return None; + } + let mut depth: usize = 0; + let mut quote: Option = None; + let mut comma: Option = None; + for (offset, character) in text[open..].char_indices() { + let position: usize = open + offset; + match (quote, character) { + (Some(open_quote), _) if character == open_quote => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(') => depth += 1, + (None, ',') if depth == 1 && comma.is_none() => comma = Some(position), + (None, ')') => { + depth -= 1; + if depth == 0 { + let argument_end: usize = comma.unwrap_or(position); + return Some((text[open + 1..argument_end].trim().to_owned(), position + 1)); + } + } + _ => {} + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 646741e2ad..f63bb88d55 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -22,6 +22,10 @@ pub const MISPLACED_CALL_CODE: &str = "P021"; pub const PARAMETER_REFERENCE_CODE: &str = "P022"; /// A runtime parameter a SQL macro body uses where SQL requires a literal. pub const LITERAL_REQUIRED_CODE: &str = "P023"; +/// An expression macro whose body's known type is not its declared `returns`. +pub const RETURNS_MISMATCH_CODE: &str = "P024"; +/// An expression macro body that misuses the type of an `ANY` family argument. +pub const BODY_TYPE_CODE: &str = "P025"; /// A declaration a SQL macro body uses that is not visible from the macro file. pub const BODY_SCOPE_CODE: &str = "S026"; @@ -35,11 +39,17 @@ pub(crate) const DEFAULT_KEYWORD: &str = "DEFAULT"; pub(crate) const RELATION_KEYWORD: &str = "RELATION"; pub(crate) const COLUMN_KEYWORD: &str = "COLUMN"; pub(crate) const OF_KEYWORD: &str = "OF"; +pub(crate) const TYPE_KEYWORD: &str = "TYPE"; pub(crate) const ENUM_KEYWORD: &str = "ENUM"; pub(crate) const LIST_KEYWORD: &str = "LIST"; pub(crate) const SET_KEYWORD: &str = "SET"; /// Names a parameter may not take, because `@name` already means something else. -pub(crate) const RESERVED_PARAMETER_NAMES: [&str; 5] = ["sql", "py", "const", "enum", "param"]; +pub(crate) const RESERVED_PARAMETER_NAMES: [&str; 6] = + ["sql", "py", "const", "enum", "param", CTE_ARGUMENT]; +/// The reserved call argument naming the CTEs a CTE-list macro call inserts. +pub const CTE_ARGUMENT: &str = "__cte"; +/// What joins a call's `__cte` name to the name of each CTE but the last. +pub(crate) const CTE_NAME_SEPARATOR: &str = "__"; /// Type names that would make a parameter raw, untyped text. pub(crate) const RAW_TYPE_NAMES: [&str; 3] = ["RAW", "SQL", "EXPRESSION"]; /// The type of a parameter that takes any scalar expression and checks no type. @@ -64,8 +74,6 @@ pub(crate) const CTE_NOUN: &str = "CTE"; /// Clause keywords a call position reads. pub(crate) const WITH_KEYWORD: &str = "WITH"; pub(crate) const SELECT_KEYWORD: &str = "SELECT"; -/// The keyword before an alias. -pub(crate) const AS_KEYWORD: &str = "AS"; /// Punctuation and operators the scans compare against. pub(crate) const COMMA: &str = ","; pub(crate) const AT_SIGN: &str = "@"; @@ -101,7 +109,14 @@ pub(crate) const PROBE_TAIL_ENDS: [&str; 11] = [ pub(crate) const INFERRED_TEXT_TYPE: &str = "TEXT"; pub(crate) const VARCHAR_TYPE: &str = "VARCHAR"; pub(crate) const FROM_KEYWORD: &str = "FROM"; -/// The projection name of an unexpanded star in a probe's analysis. +/// Words that end a select item rather than alias it. +pub(crate) const ITEM_ENDS: [&str; 4] = ["FROM", "WHERE", "UNION", "ORDER"]; +/// The relation whose columns stand in for a macro's arguments while its body is typed. +pub(crate) const STAND_IN_RELATION: &str = "__sqlbuild_macro_arguments"; +/// The polyglot validation error codes, by number, of operands and arguments of a wrong type. +pub(crate) const TYPE_ERROR_CODES: (usize, usize) = (210, 219); +/// The polyglot validation warning codes, by number, of implicit conversions. +pub(crate) const TYPE_WARNING_CODES: (usize, usize) = (210, 216); pub(crate) const STAR_PROJECTION: &str = "*"; /// The type hint analysis reports for a column whose type it does not know. pub(crate) const UNKNOWN_TYPE_HINT: &str = "UNKNOWN"; @@ -110,3 +125,12 @@ pub(crate) const ALIAS_PRECEDING_KEYWORDS: [&str; 13] = [ "FROM", "JOIN", "ON", "AND", "OR", "NOT", "WHERE", "USING", "LATERAL", "IN", "IS", "BY", "SELECT", ]; +/// Dialects tried in turn to judge whether typed markers keep a query parseable. +pub(crate) const MARKER_PARSE_DIALECTS: [polyglot_sql::DialectType; 4] = [ + polyglot_sql::DialectType::Generic, + polyglot_sql::DialectType::DuckDB, + polyglot_sql::DialectType::Snowflake, + polyglot_sql::DialectType::BigQuery, +]; +/// The literal a typed marker still types, though it is one bare word. +pub(crate) const NULL_KEYWORD: &str = "NULL"; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs index 32277f5163..5f36056974 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs}; use crate::sql_macros::constants::NON_ASCII_BYTE; use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs new file mode 100644 index 0000000000..5836d87b82 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs @@ -0,0 +1,9 @@ +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::typing::typed_calls::byte_splices; +use crate::sql_macros::models::TextSplice; + +/// Character-offset `spans` from `source` to `output` as byte splices. +#[must_use] +pub fn byte_text_splices(source: &str, output: &str, spans: &[SplicedSpan]) -> Vec { + byte_splices(source, output, spans) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs index 4deecba367..b6acd14352 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs @@ -1,4 +1,6 @@ -use crate::sql_macros::_helpers::probes::{ProbePlan, analysed_probes, check_findings, probe_plan}; +use crate::sql_macros::_helpers::typing::probes::{ + ProbePlan, analysed_probes, check_findings, probe_plan, +}; use crate::sql_macros::models::SqlMacroFinding; use crate::sql_macros::types::{ArgumentCheckInput, ProbeColumns}; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs index abf2e2520f..a7139e3c65 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs @@ -1,22 +1,27 @@ -use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; use crate::sql_macros::constants::INVALID_DEFINITION_CODE; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; -use crate::sql_macros::models::{InsertedNames, SqlMacroCall, TextContext}; +use crate::sql_macros::models::{ExpandedText, InsertedNames, SqlMacroCall, TextContext}; use crate::sql_macros::types::SqlMacroHost; -/// One outer SQL macro call's replacement text, and the names inserted so far. +/// One outer SQL macro call's replacement text with its typed calls, and the names inserted so far. pub fn expand_sql_macro_call( host: &H, context: &TextContext<'_>, inserted: InsertedNames, call: &SqlMacroCall, -) -> Result<(String, InsertedNames), ExpansionFailure> { +) -> Result<(ExpandedText, InsertedNames), ExpansionFailure> { let scanned: Vec = tokens(context.text).map_err(|_| { SqlMacroError::new( INVALID_DEFINITION_CODE, "SQL has an unclosed quote or comment", ) })?; - crate::sql_macros::_helpers::expansion::expand_call(host, context, (&scanned, inserted), call) + crate::sql_macros::_helpers::rendering::expansion::expand_call( + host, + context, + (&scanned, inserted), + call, + ) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs b/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs new file mode 100644 index 0000000000..43e8bc552d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs @@ -0,0 +1,9 @@ +use std::borrow::Cow; + +use crate::sql_macros::_helpers::typing::typed_calls::without_marker_calls; + +/// `text` with each analysis-only typed marker call replaced by the expression it typed. +#[must_use] +pub fn marker_free_text(text: &str) -> Cow<'_, str> { + without_marker_calls(text) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index ab74c41019..45b48d9884 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -1,14 +1,18 @@ pub mod bare_macro_calls; +pub mod byte_text_splices; pub mod check_sql_macro_arguments; pub mod declaration_visible_from_macro; pub mod expand_sql_macro_call; +pub mod marker_free_text; +pub mod moved_typed_calls; pub mod parse_sql_macro; pub mod render_inline_collection; +pub mod spliced_typed_calls; pub mod sql_macro_body_form; pub mod sql_macro_calls; pub mod sql_macro_cycle; -pub mod sql_macro_declared_outputs; -pub mod sql_macro_declared_type; -pub mod sql_macro_output_names; pub mod sql_macro_parameter_spans; +pub mod typed_analysis_pass; +pub mod typed_analysis_sql; +pub mod untyped_call_outputs; pub mod with_list_sql_macro_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs new file mode 100644 index 0000000000..caff6c30c2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs @@ -0,0 +1,8 @@ +use crate::sql_macros::_helpers::typing::typed_calls::spliced; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` moved by byte `splices`, sorted and non-overlapping; a call a splice cuts is dropped. +#[must_use] +pub fn moved_typed_calls(calls: Vec, splices: &[TextSplice]) -> Vec { + spliced(calls, splices) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs b/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs index cd9d71100a..190aa37629 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::definition::parsed_definition; +use crate::sql_macros::_helpers::parsing::definition::parsed_definition; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::SqlMacroDefinition; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs b/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs index 5f02404a7e..8d20f12559 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs @@ -1,5 +1,5 @@ -use crate::sql_macros::_helpers::arguments::{collection_items, collection_problem}; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::rendering::arguments::{collection_items, collection_problem}; use crate::sql_macros::constants::ARGUMENT_CODE; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::TypeFamily; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs new file mode 100644 index 0000000000..14dfa9fd39 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs @@ -0,0 +1,17 @@ +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::typing::typed_calls::{byte_splices, spliced}; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` in `source` moved to `output` by character-offset `spans`; a call a span cuts is dropped. +#[must_use] +pub fn spliced_typed_calls( + calls: Vec, + (source, output): (&str, &str), + spans: &[SplicedSpan], +) -> Vec { + if calls.is_empty() { + return calls; + } + let splices: Vec = byte_splices(source, output, spans); + spliced(calls, &splices) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs index 62db814bb3..91e989505d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs @@ -1,7 +1,7 @@ //! The form of a SQL macro body, as its definition reads it. -use crate::sql_macros::_helpers::definition::body_form; -use crate::sql_macros::_helpers::lexing::tokens; +use crate::sql_macros::_helpers::parsing::definition::body_form; +use crate::sql_macros::_helpers::parsing::lexing::tokens; use crate::sql_macros::models::SqlMacroForm; /// Whether `body` is a CTE list, a column list or an expression, from its first token. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs index 3af9c1578f..aa5ad4a182 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs @@ -6,5 +6,5 @@ pub fn sql_macro_calls(text: &str) -> Result, SqlMacroError> { if !text.contains(crate::sql_macros::constants::SQL_CALL_PREFIX) { return Ok(Vec::new()); } - crate::sql_macros::_helpers::expansion::sql_macro_calls(text) + crate::sql_macros::_helpers::rendering::expansion::sql_macro_calls(text) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs deleted file mode 100644 index f9c379d720..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::sql_macros::models::{CallScope, DeclaredOutput, SqlMacroCheck}; - -/// The select items, in any set-operation branch, of a model's top-level queries that are one call. -#[must_use] -pub fn sql_macro_declared_outputs(checks: &[SqlMacroCheck]) -> Vec { - checks - .iter() - .filter_map(|check| { - let output = check.output.as_ref()?; - let cte: Option = match &check.scope { - CallScope::Main => None, - CallScope::Cte(name) => Some(name.clone()), - CallScope::WithList => return None, - }; - Some(DeclaredOutput { - cte, - name: output.name.clone(), - returns: output.returns.clone(), - branch: output.branch, - }) - }) - .collect() -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs deleted file mode 100644 index 2e7aa03ef5..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::constants::UNKNOWN_SQL_TYPE; -use crate::sql_macros::models::DeclaredOutput; -use crate::type_system::main::normalize_type::normalize_type; -use crate::type_system::models::NormalizedType; - -/// The declared `returns` type an item takes over its inferred type: unknown or identical. -#[must_use] -pub fn sql_macro_declared_type( - declared: &[DeclaredOutput], - (cte, branch, name): (Option<&str>, usize, &str), - inferred: Option<&str>, - dialect: &str, -) -> Option { - let output: &DeclaredOutput = declared.iter().find(|output| { - output.name.eq_ignore_ascii_case(name) - && output.branch == branch - && match (&output.cte, cte) { - (None, None) => true, - (Some(declared_cte), Some(cte)) => declared_cte.eq_ignore_ascii_case(cte), - _ => false, - } - })?; - let Some(inferred) = inferred.filter(|inferred| { - !inferred.is_empty() && !inferred.eq_ignore_ascii_case(UNKNOWN_SQL_TYPE) - }) else { - return Some(output.returns.clone()); - }; - let Some(inferred_type) = normalized(inferred, dialect) else { - return Some(output.returns.clone()); - }; - (normalized(&output.returns, dialect) == Some(inferred_type)).then(|| output.returns.clone()) -} - -/// `data_type` as compared, or `None` where it does not normalise. -fn normalized(data_type: &str, dialect: &str) -> Option { - match normalize_type(data_type, dialect) { - Ok(normalization) => Some(normalization.normalized), - Err(_unnormalizable) => None, - } -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs deleted file mode 100644 index 5e06e4bbf5..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::sql_macros::_helpers::expansion::sql_macro_calls; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::placement::{CallSite, QueryParts, top_level_parts}; -use crate::sql_macros::models::{CallScope, SqlMacroCall}; - -/// The `(top-level CTE, branch, alias)` of each select item of `text` that is one SQL macro call. -#[must_use] -pub fn sql_macro_output_names(text: &str) -> Vec<(Option, usize, String)> { - let Ok(calls) = sql_macro_calls(text) else { - return Vec::new(); - }; - if calls.is_empty() { - return Vec::new(); - } - let Ok(scanned) = tokens(text) else { - return Vec::new(); - }; - let Some(parts) = top_level_parts(text, &scanned) else { - return Vec::new(); - }; - calls - .iter() - .filter(|call| !call.in_python) - .filter_map(|call| whole_item(text, (&scanned, &parts), call)) - .collect() -} - -/// The `(top-level CTE, branch, alias)` of `call` when it is a whole aliased select item. -fn whole_item( - text: &str, - (scanned, parts): (&[Token], &QueryParts), - call: &SqlMacroCall, -) -> Option<(Option, usize, String)> { - let cte: Option = match parts.scope_of(call.start) { - CallScope::Main => None, - CallScope::Cte(name) => Some(name), - CallScope::WithList => return None, - }; - let branch: usize = parts.branch_of(text, scanned, call.start)?; - CallSite::new(text, scanned, (call.start, call.end), call.replaced_end) - .whole_item_alias() - .map(|alias| (cte, branch, alias)) -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs index 419d4fcaff..58a8a05f64 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs @@ -1,7 +1,7 @@ //! The byte spans of a SQL macro body's `@name` parameter references, as expansion reads them. -use crate::sql_macros::_helpers::lexing::tokens; -use crate::sql_macros::_helpers::parameters::parameter_references; +use crate::sql_macros::_helpers::parsing::lexing::tokens; +use crate::sql_macros::_helpers::parsing::parameters::parameter_references; /// The `start..end` byte span of each `@name` in `body` that expansion replaces by an argument. #[must_use] diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs new file mode 100644 index 0000000000..ed93d049cc --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs @@ -0,0 +1,35 @@ +use crate::semantic_validation::models::Expansion; +use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; +use crate::sql_macros::models::TypedCall; + +/// `sql` with its typed calls marked and the pass mapping offsets back, if any call is marked. +#[must_use] +pub fn typed_analysis_pass( + sql: &str, + calls: &[TypedCall], + dialect: &str, +) -> Option<(String, Vec)> { + let (typed, insertions) = typed_sql(sql, &markable_calls(sql, calls, dialect)); + if insertions.is_empty() { + return None; + } + let mut counted: ((usize, usize), (usize, usize)) = ((0, 0), (0, 0)); + let characters = |text: &str, (byte, count): &mut (usize, usize), to: usize| { + *count += text[*byte..to].chars().count(); + *byte = to; + *count + }; + let mut pass: Vec = Vec::with_capacity(insertions.len()); + for insertion in &insertions { + let source: usize = characters(sql, &mut counted.0, insertion.source.0); + let output_start: usize = characters(&typed, &mut counted.1, insertion.output.0); + let output_end: usize = characters(&typed, &mut counted.1, insertion.output.1); + pass.push(Expansion { + source_start: source, + source_end: source, + output_start, + output_end, + }); + } + Some((typed, pass)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs new file mode 100644 index 0000000000..efbfd529bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs @@ -0,0 +1,12 @@ +use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `sql` with its typed calls wrapped in analysis-only markers, and each marker insertion. +#[must_use] +pub fn typed_analysis_sql( + sql: &str, + calls: &[TypedCall], + dialect: &str, +) -> (String, Vec) { + typed_sql(sql, &markable_calls(sql, calls, dialect)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs new file mode 100644 index 0000000000..cc4c9115da --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs @@ -0,0 +1,57 @@ +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::constants::ITEM_ENDS; +use crate::sql_macros::models::TypedCall; + +/// The `(output name, macro name)` of each select item that is one untyped macro call. +#[must_use] +pub fn untyped_call_outputs(sql: &str, calls: &[TypedCall]) -> Vec<(String, String)> { + let untyped: Vec<&TypedCall> = calls + .iter() + .filter(|call| call.returns.is_none() && call.returns_like.is_none()) + .collect(); + if untyped.is_empty() { + return Vec::new(); + } + let Ok(scanned) = tokens(sql) else { + return Vec::new(); + }; + let mut outputs: Vec<(String, String)> = Vec::new(); + for call in untyped { + let Some(first) = scanned.iter().position(|token| token.start == call.start) else { + continue; + }; + let opens_item: bool = first.checked_sub(1).is_some_and(|before| { + let token: &Token = &scanned[before]; + token.is(sql, ",") || token.is_word(sql, "SELECT") || token.is_word(sql, "DISTINCT") + }); + let Some(after) = scanned.iter().position(|token| token.start >= call.end) else { + continue; + }; + if !opens_item { + continue; + } + let alias: usize = if scanned[after].is_word(sql, "AS") { + after + 1 + } else { + after + }; + let Some(name) = scanned.get(alias).filter(|token| { + matches!(token.kind, TokenKind::Word | TokenKind::Quoted) && !ends_item(token, sql) + }) else { + continue; + }; + let closes_item: bool = scanned + .get(alias + 1) + .is_none_or(|next| next.is(sql, ",") || next.kind == TokenKind::Word); + if closes_item { + outputs.push((unquoted(name.text(sql)), call.macro_name.clone())); + } + } + outputs +} + +/// Whether `token` is a word that ends a select item rather than aliases it. +fn ends_item(token: &Token, sql: &str) -> bool { + ITEM_ENDS.iter().any(|word| token.is_word(sql, word)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs index 24fc2bf9f8..9669935131 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs @@ -1,5 +1,5 @@ -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::placement::CallSite; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::rendering::placement::CallSite; use crate::sql_macros::main::sql_macro_calls::sql_macro_calls; use crate::sql_macros::models::{CallPosition, SqlMacroCall}; diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 8d60199c21..7906f7aad1 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -29,8 +29,8 @@ impl SqlMacroForm { pub enum ParameterKind { /// Any SQL expression of a compatible type. Expression { data_type: String }, - /// Any scalar SQL expression, whatever its type. - Any, + /// Any scalar SQL expression, whatever its type, or any of one type family. + Any { family: Option }, /// A literal or a scalar constant. Constant { data_type: String }, /// An inline list or set, or a list or set constant; `set` for `SET`. @@ -48,7 +48,10 @@ impl ParameterKind { pub fn label(&self) -> String { match self { Self::Expression { data_type } => data_type.clone(), - Self::Any => "ANY".to_owned(), + Self::Any { family: None } => "ANY".to_owned(), + Self::Any { + family: Some(family), + } => format!("ANY {}", family.keyword()), Self::Constant { data_type } => format!("{data_type} CONST"), Self::Collection { set, data_type } => { format!("{} {data_type}", if *set { "SET" } else { "LIST" }) @@ -66,6 +69,66 @@ impl ParameterKind { } } +/// A type family an `ANY` parameter may be narrowed to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnyFamily { + /// Integer, decimal and float types. + Numeric, + /// String types. + Text, + /// Date, time and timestamp types. + Temporal, +} + +impl AnyFamily { + /// Every family, in the order messages list them. + pub const ALL: [Self; 3] = [Self::Numeric, Self::Text, Self::Temporal]; + + /// The word after `ANY` that declares the family. + pub fn keyword(self) -> &'static str { + match self { + Self::Numeric => "NUMERIC", + Self::Text => "TEXT", + Self::Temporal => "TEMPORAL", + } + } + + /// The family's widest member, which a macro body is type-checked against. + pub fn widest(self) -> &'static str { + match self { + Self::Numeric => "DOUBLE", + Self::Text => "VARCHAR", + Self::Temporal => "TIMESTAMP", + } + } + + /// What the family's values are, as messages name them. + pub fn noun(self) -> &'static str { + match self { + Self::Numeric => "a number", + Self::Text => "text", + Self::Temporal => "a date, time or timestamp", + } + } + + /// Whether a value of `family` belongs to this family; unknown and NULL values do. + pub(crate) fn admits(self, family: TypeFamily) -> bool { + matches!( + (self, family), + (_, TypeFamily::Unknown | TypeFamily::Null) + | ( + Self::Numeric, + TypeFamily::Integer | TypeFamily::Decimal | TypeFamily::Float + ) + | (Self::Text, TypeFamily::Text) + | ( + Self::Temporal, + TypeFamily::Date | TypeFamily::Timestamp | TypeFamily::Time + ) + ) + } +} + /// One declared parameter. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SqlMacroParameter { @@ -88,6 +151,8 @@ pub struct SqlMacroDefinition { pub parameters: Vec, /// The declared result type of an expression macro. pub returns: Option, + /// The parameter whose argument's type an expression macro returns, by `returns TYPE OF`. + pub returns_like: Option, pub form: SqlMacroForm, /// The body as written, after the header. pub body: String, @@ -164,6 +229,8 @@ pub struct RenderedArgument { pub parameter: usize, pub sql: String, pub shape: ArgumentShape, + /// The expression macro calls `sql` holds. + pub calls: Vec, } /// One expansion's SQL and the names it defines, for clash checks at the call site. @@ -173,6 +240,46 @@ pub struct BodyExpansion { pub sql: String, /// The CTE names a CTE-list macro defines, or the column names a column-list macro outputs. pub names: Vec, + /// The expression macro calls the SQL holds, from the body and the arguments. + pub calls: Vec, + /// The SQL of the argument whose type an expression macro call returns, by `returns TYPE OF`. + pub like: Option, +} + +/// One expression macro call's expansion in a text: its byte range and the macro's `returns`. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TypedCall { + pub start: usize, + pub end: usize, + pub macro_name: String, + /// The macro's declared result type; `None` when it declares none. + pub returns: Option, + /// The SQL of the argument whose type the call returns, by `returns TYPE OF`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub returns_like: Option, +} + +/// SQL an expansion produced and the expression macro calls it holds. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExpandedText { + pub sql: String, + /// Each expression macro call in `sql`, inner calls before the calls holding them. + pub calls: Vec, +} + +/// A replaced byte range of a text, and the byte range its replacement fills in the output. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TextSplice { + pub source: (usize, usize), + pub output: (usize, usize), +} + +/// A text with its `@const` and `@enum` references rendered, and where each one landed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RenderedDeclarations { + pub sql: String, + /// The rendered references in text order. + pub splices: Vec, } /// Where in the caller a call stands, as the text around it shows. @@ -256,31 +363,6 @@ pub struct SqlMacroCheck { pub arguments: Vec, /// The call's authored file location and where its text ends, when the host knows them. pub call: Option<(SqlMacroLocation, (usize, usize))>, - /// The output column an expression call is the whole select-list item of, and its type. - pub output: Option, -} - -/// A select item that is one expression macro call, typed by the macro's `returns`. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct DeclaredOutput { - /// The top-level CTE the item is in, or `None` for the main query. - pub cte: Option, - /// The item's alias, unquoted. - pub name: String, - pub returns: String, - /// The select it stands in: 0, or its branch of a set operation, left to right. - pub branch: usize, -} - -/// A select-list item that is one expression macro call under an alias, as `@sql.m(x) AS name`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TypedOutput { - /// The alias, unquoted. - pub name: String, - /// The macro's declared `returns` type. - pub returns: String, - /// The select the item stands in: 0, or its branch of a set operation, left to right. - pub branch: usize, } /// The top-level part of the caller's query a call stands in. @@ -314,6 +396,8 @@ pub enum ArgumentExpectation { ColumnOf { relation: String }, /// An expression of a type compatible with this one. Typed { data_type: String }, + /// An expression of a type in this family. + Family { family: AnyFamily }, } /// A call argument the caller's analysed SQL shows does not fit its parameter. @@ -325,3 +409,17 @@ pub struct SqlMacroFinding { pub call_end: (usize, usize), pub declared_at: SqlMacroLocation, } + +/// What type inference knows of an expression macro body over its declared arguments. +#[derive(Clone, Debug)] +pub(crate) enum BodyType { + /// A completely known type. + Known(String), + /// A type with unknown components, as shown, and as annotation typed it. + PartlyKnown { + shown: String, + data_type: polyglot_sql::expressions::DataType, + }, + /// Nothing, or only a guess. + Unknown, +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index ef65d03c09..31340ce436 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -4,7 +4,7 @@ mod helpers; mod test_definitions; #[path = "tests/test_expansion.rs"] mod test_expansion; -#[path = "tests/test_output_types.rs"] -mod test_output_types; +#[path = "tests/test_typed_calls.rs"] +mod test_typed_calls; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index ec8f644996..fafd5775b0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use crate::sql_macros::_helpers::expansion::expand_text; +use crate::sql_macros::_helpers::rendering::expansion::expand_text; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; use crate::sql_macros::models::{ - CallScope, SqlMacroCheck, SqlMacroDefinition, TextContext, TypedOutput, + ExpandedText, RenderedDeclarations, SqlMacroDefinition, TextContext, TextSplice, TypedCall, }; use crate::sql_macros::tests::test_types::{AsciiArguments, FakeHost}; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -53,11 +53,28 @@ impl SqlMacroHost for FakeHost { &self, _scope: &SqlMacroScope, text: &str, - ) -> Result> { - let mut rendered: String = text.to_owned(); - for (reference, value) in DECLARATIONS { - rendered = rendered.replace(reference, value); + ) -> Result> { + let mut rendered: RenderedDeclarations = RenderedDeclarations::default(); + let mut cursor: usize = 0; + while let Some((start, reference, value)) = DECLARATIONS + .iter() + .filter_map(|(reference, value)| { + text[cursor..] + .find(reference) + .map(|found| (cursor + found, *reference, *value)) + }) + .min_by_key(|(start, _, _)| *start) + { + rendered.sql.push_str(&text[cursor..start]); + let output_start: usize = rendered.sql.len(); + rendered.sql.push_str(value); + rendered.splices.push(TextSplice { + source: (start, start + reference.len()), + output: (output_start, rendered.sql.len()), + }); + cursor = start + reference.len(); } + rendered.sql.push_str(&text[cursor..]); Ok(rendered) } @@ -96,6 +113,15 @@ pub(super) fn expanded_for( macros: &[(&str, &str)], sql: &str, ) -> Result { + expanded_text_for(adapter, macros, sql).map(|expanded| expanded.sql) +} + +/// `sql` expanded with `macros` for `adapter` with its typed calls, or the error it raised. +pub(super) fn expanded_text_for( + adapter: &'static str, + macros: &[(&str, &str)], + sql: &str, +) -> Result { let host: FakeHost = FakeHost::new(macros, adapter); let context = TextContext { text: sql, @@ -129,18 +155,34 @@ impl sqlbuild_render::macro_arguments::types::ArgumentHost for AsciiArguments { } } -/// A check recording output `label` of `shout()`, typed VARCHAR, at `scope`'s `branch`. -pub(super) fn typed_check(scope: CallScope, branch: usize) -> SqlMacroCheck { - SqlMacroCheck { - macro_name: "shout".to_owned(), - scope, - plain_select: true, - arguments: Vec::new(), - call: None, - output: Some(TypedOutput { - name: "label".to_owned(), - returns: "VARCHAR".to_owned(), - branch, - }), - } +/// Each call as `macro returns: text`. +pub(super) fn shown_typed_calls(sql: &str, calls: &[TypedCall]) -> Vec { + calls + .iter() + .map(|call| { + format!( + "{} {}: {}", + call.macro_name, + call.returns.as_deref().unwrap_or("-"), + &sql[call.start..call.end] + ) + }) + .collect() +} + +/// An untyped `shout` call of each text in `sql`. +pub(super) fn untyped_calls(sql: &str, texts: &[&str]) -> Vec { + texts + .iter() + .map(|text| { + let start: usize = sql.find(text).unwrap_or_default(); + TypedCall { + returns_like: None, + start, + end: start + text.len(), + macro_name: "shout".to_owned(), + returns: None, + } + }) + .collect() } diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs index c63694a6a4..fc7b05b3df 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs @@ -130,12 +130,6 @@ fn given_invalid_macro_files_when_parsing_then_each_is_rejected() { expected_code: "P014", expected_message: "macro call @other() in SQL macro m() does not say which kind of macro it calls", }, - DefinitionErrorTestCase { - description: "expression without returns", - source: "MACRO (description \"x\", arguments (a BIGINT));\na + 1", - expected_code: "P015", - expected_message: "expression macro m() needs returns ", - }, DefinitionErrorTestCase { description: "CTE list with returns", source: "MACRO (description \"x\", returns BIGINT);\nWITH a AS (SELECT 1)", @@ -202,6 +196,36 @@ fn given_invalid_macro_files_when_parsing_then_each_is_rejected() { expected_code: "P015", expected_message: "argument n ANY takes nothing after it", }, + DefinitionErrorTestCase { + description: "ANY family with CONST", + source: "MACRO (description \"x\", arguments (n ANY NUMERIC CONST), returns BIGINT);\n@n", + expected_code: "P015", + expected_message: "argument n ANY NUMERIC takes nothing after it", + }, + DefinitionErrorTestCase { + description: "LIST of an ANY family", + source: "MACRO (description \"x\", arguments (codes LIST ANY TEXT), returns BOOLEAN);\n'a' IN @codes", + expected_code: "P015", + expected_message: "argument codes LIST ANY: ANY is for scalar arguments only", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF an unknown argument", + source: "MACRO (description \"x\", arguments (v ANY), returns TYPE OF w);\n@v", + expected_code: "P015", + expected_message: "returns TYPE OF w names no argument of macro m()", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF a LIST argument", + source: "MACRO (description \"x\", arguments (v LIST VARCHAR), returns TYPE OF v);\n'a' IN @v", + expected_code: "P015", + expected_message: "returns TYPE OF v names a LIST VARCHAR argument of macro m(); only a scalar argument has a type to return", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF on a CTE list", + source: "MACRO (description \"x\", arguments (src RELATION (id BIGINT)), returns TYPE OF src);\nWITH a AS (SELECT * FROM @src)", + expected_code: "P015", + expected_message: "CTE-list macro m() cannot declare returns", + }, DefinitionErrorTestCase { description: "LIST of ANY", source: "MACRO (description \"x\", arguments (codes LIST ANY), returns BOOLEAN);\n'a' IN @codes", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index 6307803193..c15ba68538 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -9,7 +9,7 @@ use crate::sql_macros::tests::test_types::{ const CENTS_TO_DOLLARS: (&str, &str) = ( "cents_to_dollars", - "MACRO (\n description \"Cents to dollars\",\n arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n returns DECIMAL(18, 2),\n);\n\nROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places)\n", + "MACRO (\n description \"Cents to dollars\",\n arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n returns DECIMAL(18, 2),\n);\n\nCAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2))\n", ); const NET_AMOUNT: (&str, &str) = ( "net_amount", @@ -121,6 +121,30 @@ const ORDER_KEY: (&str, &str) = ( "order_key", "MACRO (description \"Order key\", arguments (store_code VARCHAR, order_number ANY), returns VARCHAR);\n\n@store_code || '::' || CAST(@order_number AS VARCHAR)\n", ); +const SHOUTED_WRONG: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\nUPPER(@value)\n", +); +const SHOUTED_UNTYPED: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value VARCHAR));\n\nUPPER(@value)\n", +); +const SHOUTED_ANY: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value ANY), returns BIGINT);\n\n@value\n", +); +const SHOUTED_NESTED: (&str, &str) = ( + "shouted_twice", + "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\n@sql.plus_one(LENGTH(@value)) || 'x'\n", +); +const ROUND_ANY: (&str, &str) = ( + "round_any", + "MACRO (description \"Rounded\", arguments (v ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF v);\n\nROUND(@v, @places)\n", +); +const SHOUT_NUMBER: (&str, &str) = ( + "shout_number", + "MACRO (description \"Shouted\", arguments (v ANY NUMERIC));\n\nUPPER(@v)\n", +); const ODD: (&str, &str) = ( "odd", "MACRO (description \"Odd\", arguments (v FOO BAR), returns BIGINT);\n\n@v + 1\n", @@ -134,21 +158,21 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { adapter: "", macros: &[CENTS_TO_DOLLARS], sql: "SELECT @sql.cents_to_dollars(o.fee_cents + o.tax_cents, places := 0) AS fees FROM orders AS o", - expected_sql: "SELECT ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees FROM orders AS o", + expected_sql: "SELECT CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM orders AS o", }, ExpansionTestCase { description: "expression stacking another expression macro", adapter: "", macros: &[CENTS_TO_DOLLARS, NET_AMOUNT], sql: "SELECT @sql.net_amount(o.gross_cents, o.refund_cents) AS net FROM orders AS o", - expected_sql: "SELECT ROUND(CAST((o.gross_cents - COALESCE(o.refund_cents, 0)) AS DECIMAL(18, 4)) / 100, 2) AS net FROM orders AS o", + expected_sql: "SELECT CAST(ROUND((o.gross_cents - COALESCE(o.refund_cents, 0)) / 100, 2) AS DECIMAL(18, 2)) AS net FROM orders AS o", }, ExpansionTestCase { description: "a backslash-escaped quote in a BigQuery string is not a call", adapter: "bigquery", macros: &[NET_AMOUNT, CENTS_TO_DOLLARS], sql: "SELECT 'it\\'s @sql.x(' AS note, @sql.net_amount(o.a, o.b) AS net FROM orders AS o", - expected_sql: "SELECT 'it\\'s @sql.x(' AS note, ROUND(CAST((o.a - COALESCE(o.b, 0)) AS DECIMAL(18, 4)) / 100, 2) AS net FROM orders AS o", + expected_sql: "SELECT 'it\\'s @sql.x(' AS note, CAST(ROUND((o.a - COALESCE(o.b, 0)) / 100, 2) AS DECIMAL(18, 2)) AS net FROM orders AS o", }, ExpansionTestCase { description: "an expression call before AT TIME ZONE is parenthesised", @@ -192,6 +216,27 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\n@sql.clean_orders(orders_raw)\nfinal AS (SELECT order_id, amount FROM latest)\nSELECT * FROM final", expected_sql: "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\nlive AS (SELECT * FROM orders_raw WHERE NOT is_deleted),\nranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM live\n),\nlatest AS (SELECT * EXCLUDE (row_num) FROM ranked WHERE row_num = 1),\nfinal AS (SELECT order_id, amount FROM latest)\nSELECT * FROM final", }, + ExpansionTestCase { + description: "CTE list called twice, each call naming its CTEs with __cte", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH eu_raw AS (SELECT 1 AS order_id), us_raw AS (SELECT 2 AS order_id),\n@sql.clean_orders(eu_raw, __cte := eu_orders)\n@sql.clean_orders(us_raw, __CTE := us_orders)\nfinal AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders)\nSELECT * FROM final", + expected_sql: "WITH eu_raw AS (SELECT 1 AS order_id), us_raw AS (SELECT 2 AS order_id),\neu_orders__live AS (SELECT * FROM eu_raw WHERE NOT is_deleted),\neu_orders__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM eu_orders__live\n),\neu_orders AS (SELECT * EXCLUDE (row_num) FROM eu_orders__ranked WHERE row_num = 1),\nus_orders__live AS (SELECT * FROM us_raw WHERE NOT is_deleted),\nus_orders__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM us_orders__live\n),\nus_orders AS (SELECT * EXCLUDE (row_num) FROM us_orders__ranked WHERE row_num = 1),\nfinal AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders)\nSELECT * FROM final", + }, + ExpansionTestCase { + description: "__cte renames the macro's CTEs but never an argument of the same name", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH live AS (SELECT 1 AS order_id), @sql.clean_orders(live, __cte := picked) SELECT * FROM picked", + expected_sql: "WITH live AS (SELECT 1 AS order_id), picked__live AS (SELECT * FROM live WHERE NOT is_deleted),\npicked__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM picked__live\n),\npicked AS (SELECT * EXCLUDE (row_num) FROM picked__ranked WHERE row_num = 1) SELECT * FROM picked", + }, + ExpansionTestCase { + description: "__cte renames qualified references to a renamed CTE", + adapter: "", + macros: &[LATEST_PER_KEY], + sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), @sql.latest_per_key(r, id, __cte := l) SELECT * FROM l", + expected_sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), l__latest_ranked AS (\n SELECT r.*, ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) AS row_num FROM r\n),\nl AS (SELECT * EXCLUDE (row_num) FROM l__latest_ranked WHERE row_num = 1) SELECT * FROM l", + }, ExpansionTestCase { description: "CTE list extracted from a model compiles to the original", adapter: "", @@ -281,7 +326,7 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { adapter: "", macros: &[CENTS_TO_DOLLARS], sql: "SELECT @py.label(@sql.cents_to_dollars(o.fee)) AS fee FROM orders AS o", - expected_sql: "SELECT @py.label(\"ROUND(CAST(o.fee AS DECIMAL(18, 4)) / 100, 2)\") AS fee FROM orders AS o", + expected_sql: "SELECT @py.label(\"CAST(ROUND(o.fee / 100, 2) AS DECIMAL(18, 2))\") AS fee FROM orders AS o", }, ExpansionTestCase { description: "runtime ROUND precision where the adapter allows one", @@ -388,6 +433,27 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "SELECT @sql.odd(1) AS a", expected_sql: "SELECT 1 + 1 AS a", }, + ExpansionTestCase { + description: "an ANY family argument of its family", + adapter: "duckdb", + macros: &[ROUND_ANY], + sql: "SELECT @sql.round_any(o.amount, 1) AS a FROM orders AS o", + expected_sql: "SELECT ROUND(o.amount, 1) AS a FROM orders AS o", + }, + ExpansionTestCase { + description: "an expression macro without returns", + adapter: "duckdb", + macros: &[SHOUTED_UNTYPED], + sql: "SELECT @sql.shouted(o.status) AS a FROM orders AS o", + expected_sql: "SELECT UPPER(o.status) AS a FROM orders AS o", + }, + ExpansionTestCase { + description: "an ANY argument leaves the body untyped, so returns is trusted", + adapter: "duckdb", + macros: &[SHOUTED_ANY], + sql: "SELECT @sql.shouted(o.status) AS a FROM orders AS o", + expected_sql: "SELECT o.status AS a FROM orders AS o", + }, ]; for test_case in test_cases { @@ -421,6 +487,38 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { expected_code: "P018", expected_message: "CTE \"live\" from macro clean_orders() clashes with a CTE from macro clean_orders() in model:orders_clean", }, + ExpansionErrorTestCase { + description: "__cte on an expression macro", + adapter: "", + macros: &[PLUS_ONE], + sql: "SELECT @sql.plus_one(1, __cte := x)", + expected_code: "P017", + expected_message: "__cte names the CTEs a CTE-list macro inserts; plus_one() is an expression macro", + }, + ExpansionErrorTestCase { + description: "__cte that is not a bare name", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), @sql.clean_orders(a, __cte := 'x') SELECT 1", + expected_code: "P017", + expected_message: "__cte of macro clean_orders() must be a bare CTE name; got 'x'", + }, + ExpansionErrorTestCase { + description: "the same __cte name twice", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), @sql.clean_orders(a, __cte := x) @sql.clean_orders(a, __cte := x) SELECT 1", + expected_code: "P018", + expected_message: "CTE \"x__live\" from macro clean_orders() clashes with a CTE from macro clean_orders() in model:orders_clean", + }, + ExpansionErrorTestCase { + description: "a __cte name the caller already defines", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), x AS (SELECT 2), @sql.clean_orders(a, __cte := x) SELECT 1", + expected_code: "P018", + expected_message: "CTE \"x\" from macro clean_orders() clashes with a CTE already defined in model:orders_clean", + }, ExpansionErrorTestCase { description: "column clash with the caller", adapter: "", @@ -429,6 +527,38 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { expected_code: "P018", expected_message: "column \"created_at\" from macro audit_columns() clashes with a column already defined in model:orders_clean", }, + ExpansionErrorTestCase { + description: "a body whose known type is not returns", + adapter: "duckdb", + macros: &[SHOUTED_WRONG], + sql: "SELECT @sql.shouted(o.status) FROM orders AS o", + expected_code: "P024", + expected_message: "macro shouted() declares returns BIGINT, but its body is VARCHAR", + }, + ExpansionErrorTestCase { + description: "a body typed through a nested call", + adapter: "duckdb", + macros: &[SHOUTED_NESTED, PLUS_ONE], + sql: "SELECT @sql.shouted_twice(o.status) FROM orders AS o", + expected_code: "P024", + expected_message: "macro shouted_twice() declares returns BIGINT, but its body is VARCHAR", + }, + ExpansionErrorTestCase { + description: "a literal outside an ANY family", + adapter: "duckdb", + macros: &[ROUND_ANY], + sql: "SELECT @sql.round_any('ten')", + expected_code: "P017", + expected_message: "argument v of macro round_any() expects a number; 'ten' is a string", + }, + ExpansionErrorTestCase { + description: "a body that misuses an ANY family argument", + adapter: "duckdb", + macros: &[SHOUT_NUMBER], + sql: "SELECT @sql.shout_number(o.amount) FROM orders AS o", + expected_code: "P025", + expected_message: "body of macro shout_number() does not accept every value of v ANY NUMERIC", + }, ExpansionErrorTestCase { description: "cycle", adapter: "", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs deleted file mode 100644 index 9861f19e11..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::main::sql_macro_output_names::sql_macro_output_names; -use crate::sql_macros::models::CallScope; -use crate::sql_macros::tests::helpers::typed_check; -use crate::sql_macros::tests::test_types::{OutputNamesTestCase, OutputTypeTestCase}; - -#[test] -fn given_a_typed_macro_output_when_typing_the_column_then_returns_fills_unknown_and_refines() { - let test_cases = [ - OutputTypeTestCase { - description: "unknown inferred type takes returns", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", None), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "an identical type takes returns' spelling", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "LABEL", Some("TEXT")), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a different known family keeps the inferred type", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("BIGINT")), - expected_type: None, - }, - OutputTypeTestCase { - description: "a longer text type stays", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("VARCHAR(100)")), - expected_type: None, - }, - OutputTypeTestCase { - description: "the unknown placeholder takes returns", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("UNKNOWN")), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a known time type is not unknown", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("TIME")), - expected_type: None, - }, - OutputTypeTestCase { - description: "a known array type is not unknown", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("TEXT[]")), - expected_type: None, - }, - OutputTypeTestCase { - description: "another column is untouched", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "other", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in a CTE types that CTE's item", - check: typed_check(CallScope::Cte("base".to_owned()), 0), - item: (Some("BASE"), 0, "label", None), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a call in a CTE leaves a main-query item of that name", - check: typed_check(CallScope::Cte("base".to_owned()), 0), - item: (None, 0, "label", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in another branch of a set operation leaves this one", - check: typed_check(CallScope::Main, 1), - item: (None, 0, "label", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in a later branch of a set operation types that branch", - check: typed_check(CallScope::Cte("base".to_owned()), 1), - item: (Some("base"), 1, "label", Some("TEXT")), - expected_type: Some("VARCHAR"), - }, - ]; - - for test_case in test_cases { - let (cte, branch, name, inferred) = test_case.item; - let declared = sql_macro_declared_outputs(&[test_case.check]); - let typed = sql_macro_declared_type(&declared, (cte, branch, name), inferred, "duckdb"); - - assert_eq!( - typed.as_deref(), - test_case.expected_type, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_model_sql_when_reading_sql_macro_outputs_then_only_whole_aliased_items_count() { - let test_cases = [ - OutputNamesTestCase { - description: "aliased whole items, with and without AS", - sql: "SELECT @sql.a(x) AS one, @sql.b(y) two, @sql.c(z) AS \"Three\" FROM t", - expected_names: &[(None, 0, "one"), (None, 0, "two"), (None, 0, "Three")], - }, - OutputNamesTestCase { - description: "calls inside expressions or without an alias", - sql: "SELECT @sql.a(x) || 'x' AS one, UPPER(@sql.b(y)) AS two, @sql.c(z) FROM t", - expected_names: &[], - }, - OutputNamesTestCase { - description: "a call in a CTE, and ones in each branch of a set operation", - sql: "WITH c AS (SELECT @sql.a(x) AS one FROM t) SELECT @sql.b(y) AS two FROM c UNION ALL SELECT 'x' FROM c UNION ALL SELECT @sql.c(z) AS three FROM c", - expected_names: &[(Some("c"), 0, "one"), (None, 0, "two"), (None, 2, "three")], - }, - OutputNamesTestCase { - description: "a star EXCEPT modifier is not a set operation", - sql: "SELECT * EXCEPT (a) FROM t UNION ALL SELECT @sql.b(y) AS two FROM t", - expected_names: &[(None, 1, "two")], - }, - OutputNamesTestCase { - description: "calls in subqueries, a nested WITH or beside a parenthesised branch", - sql: "WITH c AS (WITH d AS (SELECT 1 AS x) SELECT @sql.a(x) AS one FROM d), e AS (SELECT * FROM (SELECT @sql.b(y) AS two FROM t) AS s) SELECT @sql.c(z) AS three FROM c UNION ALL (SELECT 'x' FROM e)", - expected_names: &[], - }, - OutputNamesTestCase { - description: "a call in the main query after CTEs", - sql: "WITH c AS (SELECT x FROM t) SELECT x, @sql.a(x) AS one FROM c", - expected_names: &[(None, 0, "one")], - }, - ]; - - for test_case in test_cases { - let names: Vec<(Option, usize, String)> = sql_macro_output_names(test_case.sql); - let found: Vec<(Option<&str>, usize, &str)> = names - .iter() - .map(|(cte, branch, name)| (cte.as_deref(), *branch, name.as_str())) - .collect(); - - assert_eq!(found, test_case.expected_names, "{}", test_case.description); - } -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs new file mode 100644 index 0000000000..d5f3821a2b --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -0,0 +1,280 @@ +use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::main::untyped_call_outputs::untyped_call_outputs; +use crate::sql_macros::models::{ExpandedText, TypedCall}; +use crate::sql_macros::tests::helpers::{expanded_text_for, shown_typed_calls, untyped_calls}; +use crate::sql_macros::tests::test_types::{ + ExpandedAnalysisTestCase, ReservedNameTestCase, TypedAnalysisTestCase, TypedCallTestCase, + UntypedOutputsTestCase, +}; + +const SHOUT: (&str, &str) = ( + "shout", + "MACRO (description \"Shout\", arguments (value VARCHAR), returns VARCHAR);\n\nUPPER(@value)\n", +); +const ADD: (&str, &str) = ( + "add", + "MACRO (description \"Add\", arguments (a BIGINT, b BIGINT), returns BIGINT);\n\n@a + @b\n", +); +const LOUD: (&str, &str) = ( + "loud", + "MACRO (description \"Loud\", arguments (value VARCHAR), returns VARCHAR);\n\n\ + @sql.shout(@value) || '!'\n", +); +const SAME: (&str, &str) = ( + "same", + "MACRO (description \"Same\", arguments (value VARCHAR), returns VARCHAR);\n\n@value\n", +); +const NOTHING: (&str, &str) = ( + "nothing", + "MACRO (description \"Nothing\", arguments (value VARCHAR), returns BIGINT);\n\nNULL\n", +); +const PLACES: (&str, &str) = ( + "places", + "MACRO (description \"Places\", arguments (value BIGINT), returns BIGINT);\n\n\ + @value + @const(\"price_places\")\n", +); +const LIKE_VALUE: (&str, &str) = ( + "like_value", + "MACRO (description \"Like\", arguments (value ANY, fallback ANY DEFAULT NULL), \ + returns TYPE OF value);\n\nCOALESCE(@value, @fallback)\n", +); + +#[test] +fn given_expression_macro_calls_when_expanding_then_each_expansion_is_recorded_where_it_lands() { + let test_cases = [ + TypedCallTestCase { + description: "a call in the select list", + macros: &[SHOUT], + sql: "SELECT o.id, @sql.shout(o.status) AS s FROM orders AS o", + expected_calls: &["shout VARCHAR: UPPER(o.status)"], + }, + TypedCallTestCase { + description: "a call parenthesised where it stands", + macros: &[ADD], + sql: "SELECT @sql.add(o.a, o.b) * 2 AS doubled FROM orders AS o", + expected_calls: &["add BIGINT: (o.a + o.b)"], + }, + TypedCallTestCase { + description: "a nested call in a body, inner first", + macros: &[SHOUT, LOUD], + sql: "SELECT @sql.loud(o.status) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "loud VARCHAR: UPPER(o.status) || '!'", + ], + }, + TypedCallTestCase { + description: "a call in another call's argument", + macros: &[SHOUT], + sql: "SELECT @sql.shout(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "shout VARCHAR: UPPER(UPPER(o.status))", + ], + }, + TypedCallTestCase { + description: "a body that is its parameter, passed another call", + macros: &[SHOUT, SAME], + sql: "SELECT @sql.same(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "same VARCHAR: UPPER(o.status)", + ], + }, + TypedCallTestCase { + description: "calls after rendered declarations", + macros: &[SHOUT, PLACES], + sql: "SELECT @const(\"price_places\") AS p, @sql.places(o.a) AS a, \ + @sql.shout(@enum(\"order_status\").OPEN) AS s FROM orders AS o", + expected_calls: &["places BIGINT: o.a + 2", "shout VARCHAR: UPPER('open')"], + }, + TypedCallTestCase { + description: "a call in a later set-operation branch", + macros: &[NOTHING], + sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"stg_orders\") AS o\nUNION ALL\n\ + SELECT @sql.nothing(o.status) AS v FROM __ref(\"stg_orders\") AS o", + expected_calls: &["nothing BIGINT: NULL"], + }, + TypedCallTestCase { + description: "a call inside a Python macro argument is not recorded", + macros: &[SHOUT], + sql: "SELECT @py.wrap(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let expanded = expanded_text_for("duckdb", test_case.macros, test_case.sql) + .unwrap_or_else(|error| panic!("{}: {error:?}", test_case.description)); + assert_eq!( + shown_typed_calls(&expanded.sql, &expanded.calls), + test_case.expected_calls, + "{}: {}", + test_case.description, + expanded.sql + ); + } +} + +#[test] +fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns_is_marked() { + let test_cases = [ + TypedAnalysisTestCase { + description: "one call", + sql: "SELECT UPPER(x) AS s FROM t", + calls: &[(7, 15, Some("VARCHAR"))], + expected_sql: "SELECT __polyglot_typed(UPPER(x), 'VARCHAR') AS s FROM t", + }, + TypedAnalysisTestCase { + description: "a call without returns is not marked", + sql: "SELECT UPPER(x) AS s FROM t", + calls: &[(7, 15, None)], + expected_sql: "SELECT UPPER(x) AS s FROM t", + }, + TypedAnalysisTestCase { + description: "nested calls with one range, inner recorded first", + sql: "SELECT t.x FROM t", + calls: &[(7, 10, Some("INT")), (7, 10, Some("BIGINT"))], + expected_sql: "SELECT __polyglot_typed(__polyglot_typed(t.x, 'INT'), 'BIGINT') FROM t", + }, + TypedAnalysisTestCase { + description: "adjacent calls close before the next opens", + sql: "SELECT (t.a)||(t.b) FROM t", + calls: &[(7, 12, Some("INT")), (14, 19, Some("INT"))], + expected_sql: "SELECT __polyglot_typed((t.a), 'INT')||__polyglot_typed((t.b), 'INT') FROM t", + }, + TypedAnalysisTestCase { + description: "a call ending in a line comment closes it first", + sql: "SELECT t.x -- note\nFROM t", + calls: &[(7, 18, Some("DECIMAL(18, 2)"))], + expected_sql: "SELECT __polyglot_typed(t.x -- note\n, 'DECIMAL(18, 2)')\nFROM t", + }, + TypedAnalysisTestCase { + description: "a call in a keyword slot is left unmarked", + sql: "SELECT EXTRACT(YEAR FROM t.c) AS y FROM t", + calls: &[(15, 19, Some("VARCHAR"))], + expected_sql: "SELECT EXTRACT(YEAR FROM t.c) AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a bare-word call is never marked", + sql: "SELECT YEAR AS y FROM t", + calls: &[(7, 11, Some("VARCHAR"))], + expected_sql: "SELECT YEAR AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a call in a type slot is left unmarked", + sql: "SELECT CAST(t.x AS DECIMAL(18, 2)) AS y FROM t", + calls: &[(19, 33, Some("VARCHAR"))], + expected_sql: "SELECT CAST(t.x AS DECIMAL(18, 2)) AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a call whose marker would not parse is left unmarked", + sql: "SELECT (t.a)(t.b) FROM t", + calls: &[(7, 12, Some("INT")), (12, 17, Some("INT"))], + expected_sql: "SELECT (t.a)(t.b) FROM t", + }, + TypedAnalysisTestCase { + description: "a quote in returns is doubled", + sql: "SELECT t.x FROM t", + calls: &[(7, 10, Some("ENUM('a')"))], + expected_sql: "SELECT __polyglot_typed(t.x, 'ENUM(''a'')') FROM t", + }, + ]; + for test_case in test_cases { + let calls: Vec = test_case + .calls + .iter() + .map(|(start, end, returns)| TypedCall { + returns_like: None, + start: *start, + end: *end, + macro_name: "m".to_owned(), + returns: returns.map(str::to_owned), + }) + .collect(); + let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); + assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + let removed: String = { + let mut text: String = String::new(); + let mut cursor: usize = 0; + for insertion in &insertions { + text.push_str(&typed[cursor..insertion.output.0]); + cursor = insertion.output.1; + } + text.push_str(&typed[cursor..]); + text + }; + assert_eq!(removed, test_case.sql, "{}", test_case.description); + } +} + +#[test] +fn given_the_reserved_analysis_function_when_comparing_then_authored_sql_rejects_the_parsed_name() { + let test_cases = [ReservedNameTestCase { + description: "the typed marker", + authored_name: + sqlbuild_attachments::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION, + expected_parsed_name: polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION, + }]; + + for test_case in test_cases { + assert_eq!( + test_case.authored_name, test_case.expected_parsed_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_calls_without_returns_when_listing_outputs_then_only_whole_named_items_are_named() { + let test_cases = [ + UntypedOutputsTestCase { + description: "aliased whole calls are named, operands and unnamed items are not", + sql: "SELECT UPPER(a) AS loud, UPPER(b) quiet, UPPER(c) || 'x' AS joined, UPPER(d) FROM t", + call_texts: &["UPPER(a)", "UPPER(b)", "UPPER(c)", "UPPER(d)"], + expected_outputs: &[("loud", "shout"), ("quiet", "shout")], + }, + UntypedOutputsTestCase { + description: "a keyword after a call is not its alias", + sql: "SELECT id, UPPER(a) FROM t", + call_texts: &["UPPER(a)"], + expected_outputs: &[], + }, + ]; + + for test_case in test_cases { + let calls: Vec = untyped_calls(test_case.sql, test_case.call_texts); + + let outputs: Vec<(String, String)> = untyped_call_outputs(test_case.sql, &calls); + + let named: Vec<(&str, &str)> = outputs + .iter() + .map(|(output, name)| (output.as_str(), name.as_str())) + .collect(); + assert_eq!( + named, test_case.expected_outputs, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_returns_type_of_call_when_building_the_analysis_text_then_it_asserts_its_argument() { + let test_cases = [ExpandedAnalysisTestCase { + description: "the call is typed like its argument", + macros: &[LIKE_VALUE], + sql: "SELECT @sql.like_value(o.created_at + 1) AS v FROM orders AS o", + expected_sql: "SELECT __polyglot_typed_like(COALESCE((o.created_at + 1), NULL), \ + o.created_at + 1) AS v FROM orders AS o", + }]; + + for test_case in test_cases { + let expanded: ExpandedText = + expanded_text_for("duckdb", test_case.macros, test_case.sql).expect("expands"); + + let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls, "duckdb"); + + assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 5058bb8c6e..a3164977ff 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -63,27 +63,6 @@ pub(super) struct PythonArgumentTestCase { /// Argument parsing without Unicode name lookups, which these arguments never use. pub(super) struct AsciiArguments; -/// One recorded call, one analysed select item and the type the item then has, if changed. -pub(super) struct OutputTypeTestCase { - pub(super) description: &'static str, - pub(super) check: crate::sql_macros::models::SqlMacroCheck, - /// The item's CTE (`None` for the main query), set-operation branch, name and inferred type. - pub(super) item: ( - Option<&'static str>, - usize, - &'static str, - Option<&'static str>, - ), - pub(super) expected_type: Option<&'static str>, -} - -/// Model SQL and the scope and alias of each whole-item SQL macro call. -pub(super) struct OutputNamesTestCase { - pub(super) description: &'static str, - pub(super) sql: &'static str, - pub(super) expected_names: &'static [(Option<&'static str>, usize, &'static str)], -} - pub(super) struct MacroDescriptionTestCase { pub(super) description: &'static str, pub(super) source: &'static str, @@ -96,3 +75,42 @@ pub(super) struct MacroQuoteErrorTestCase { pub(super) expected_message: &'static str, pub(super) expected_location: (usize, usize), } + +/// A text expanded with its typed calls, each shown as `macro returns: text`. +pub(super) struct TypedCallTestCase { + pub(super) description: &'static str, + pub(super) macros: &'static [(&'static str, &'static str)], + pub(super) sql: &'static str, + pub(super) expected_calls: &'static [&'static str], +} + +/// A compiled text and its calls, with the analysis text the markers make of it. +pub(super) struct TypedAnalysisTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) calls: &'static [(usize, usize, Option<&'static str>)], + pub(super) expected_sql: &'static str, +} + +/// Two names that must stay equal. +pub(super) struct ReservedNameTestCase { + pub(super) description: &'static str, + pub(super) authored_name: &'static str, + pub(super) expected_parsed_name: &'static str, +} + +/// A query and the texts of its untyped calls, with the outputs each whole call names. +pub(super) struct UntypedOutputsTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) call_texts: &'static [&'static str], + pub(super) expected_outputs: &'static [(&'static str, &'static str)], +} + +/// A query calling `macros`, with the analysis text its expansion marks. +pub(super) struct ExpandedAnalysisTestCase { + pub(super) description: &'static str, + pub(super) macros: &'static [(&'static str, &'static str)], + pub(super) sql: &'static str, + pub(super) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/types.rs b/crates/sqlbuild-analysis/src/sql_macros/types.rs index 668fe2b48a..0c6fb2ae32 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/types.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use crate::sql_macros::errors::ExpansionFailure; -use crate::sql_macros::models::{SqlMacroCheck, SqlMacroDefinition}; +use crate::sql_macros::models::{RenderedDeclarations, SqlMacroCheck, SqlMacroDefinition}; /// Whose scope a piece of SQL resolves its declarations and macros in. #[derive(Clone, Debug)] @@ -33,12 +33,12 @@ pub trait SqlMacroHost { name: &str, ) -> Result, ExpansionFailure>; - /// `text` with its `@const` and `@enum` references rendered from `scope`. + /// `text` with its `@const` and `@enum` references rendered from `scope`, and where each landed. fn render_declarations( &self, scope: &SqlMacroScope, text: &str, - ) -> Result>; + ) -> Result>; /// What the constant `name` visible from `scope` holds. fn constant_shape( diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs index e37fddd4b9..1b4007841f 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs @@ -1,6 +1,8 @@ //! Python's `_normalize_with_polyglot`: the parsed type's SQL and arguments. -use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions}; +use polyglot_sql::{ + ComplexityGuardOptions, DataType, Dialect, DialectType, Expression, ParseOptions, +}; use serde_json::{Map, Value}; use crate::type_system::_helpers::fallback::{ @@ -34,6 +36,20 @@ pub(crate) fn parse_type(type_sql: &str, dialect: &Dialect) -> Result Result { + match polyglot.generate(expression) { + Ok(generated) => Ok(generated), + Err(polyglot_sql::error::Error::Unsupported { .. }) => Dialect::get(DialectType::Generic) + .generate(expression) + .map_err(|error| TypeNormalizationError::Generation(error.to_string())), + Err(error) => Err(TypeNormalizationError::Generation(error.to_string())), + } +} + /// Python's `_normalized_from_parsed_type`. pub(crate) fn normalize_parsed( parsed: DataType, @@ -42,9 +58,7 @@ pub(crate) fn normalize_parsed( ) -> Result { let args: Map = shallow_args(&parsed); let expression: Expression = Expression::DataType(parsed); - let generated: String = polyglot - .generate(&expression) - .map_err(|error| TypeNormalizationError::Generation(error.to_string()))?; + let generated: String = generated_type(&expression, polyglot)?; let normalized_name: String = upper(&generated).replace(' ', ""); let dtype_name: String = polyglot_type_name(&args); let params: Vec = polyglot_type_params(&args); diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs index 49f9fa71a8..cdb7d766da 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs @@ -1 +1,3 @@ +pub(crate) mod replacement; +pub(crate) mod reserved; pub(crate) mod scan; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs new file mode 100644 index 0000000000..ba51a73d01 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs @@ -0,0 +1,62 @@ +//! Cursor intrinsic calls replaced with the byte span of each replacement. + +use crate::cursor_intrinsics::_helpers::scan::{ + CallScan, ScanStep, call_scan, contains_intrinsic_name, step, +}; +use crate::cursor_intrinsics::models::IntrinsicSpan; +use crate::sql_lexing::main::unclosed_message::unclosed_message; + +/// `sql` with each cursor intrinsic call replaced and each call's byte span, or the call's error. +pub(crate) fn replaced_with_spans( + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, Vec), String> { + if !contains_intrinsic_name(sql) { + return Ok((sql.to_owned(), Vec::new())); + } + let mut replaced: String = String::with_capacity(sql.len()); + let mut last: usize = 0; + let mut spans: Vec = Vec::new(); + let mut index: usize = 0; + while index < sql.len() { + let (name, end): (&str, usize) = match step(sql, index) { + ScanStep::Next(next) => { + index = next; + continue; + } + ScanStep::Unclosed(construct) => return Err(unclosed_message(context, construct)), + ScanStep::Intrinsic { name, end } => (name, end), + }; + let close: usize = match call_scan(sql, end) { + CallScan::Called(close) => close, + CallScan::NotCalled => { + return Err(format!("{context} intrinsic {name} must be called with ()")); + } + CallScan::Arguments => { + return Err(format!( + "{context} intrinsic {name} does not accept arguments" + )); + } + CallScan::Unclosed(construct) => { + return Err(unclosed_message( + &format!("{context} cursor intrinsic"), + construct, + )); + } + }; + replaced.push_str(&sql[last..index]); + let output_start: usize = replaced.len(); + replaced.push_str(&replacement(name)); + spans.push(IntrinsicSpan { + source_start: index, + source_end: close + 1, + output_start, + output_end: replaced.len(), + }); + last = close + 1; + index = close + 1; + } + replaced.push_str(&sql[last..]); + Ok((replaced, spans)) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs new file mode 100644 index 0000000000..5178cc2890 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs @@ -0,0 +1,17 @@ +//! Authored SQL that names the reserved analysis function. + +use crate::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION; + +/// The error for `context` when `sql` names the reserved analysis function in any case. +pub(crate) fn reserved_function_error(sql: &str, context: &str) -> Option { + let name: &[u8] = RESERVED_ANALYSIS_FUNCTION.as_bytes(); + sql.as_bytes() + .windows(name.len()) + .any(|window| window.eq_ignore_ascii_case(name)) + .then(|| { + format!( + "{context} uses {RESERVED_ANALYSIS_FUNCTION}, a name reserved for SQL analysis; \ + rename it" + ) + }) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs new file mode 100644 index 0000000000..c9ca3318e5 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs @@ -0,0 +1,4 @@ +//! Names authored SQL may not use because analysis gives them a meaning of its own. + +/// The analysis-only type assertion the SQL parser lowers into a typed expression. +pub const RESERVED_ANALYSIS_FUNCTION: &str = "__polyglot_typed"; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs index c50e3bbc2f..570e757270 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs @@ -1,5 +1,6 @@ //! `reject_cursor_intrinsics`, with the error it raises. +use crate::cursor_intrinsics::_helpers::reserved::reserved_function_error; use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::cursor_intrinsics::models::IntrinsicCheck; @@ -14,6 +15,9 @@ pub fn intrinsic_free(sql: &str, reserved_markers: &[String], context: &str) -> "{context} contains a reserved internal cursor marker" )); } + if let Some(message) = reserved_function_error(sql, context) { + return IntrinsicCheck::Rejected(message); + } match replace_intrinsics(sql, context, str::to_owned) { Err(message) => IntrinsicCheck::Rejected(message), Ok((_, true)) => IntrinsicCheck::Rejected(format!( diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs index ac124fa97a..d23f7da39e 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs @@ -1,3 +1,4 @@ pub mod intrinsic_free; pub mod replace_intrinsics; +pub mod replace_intrinsics_with_spans; pub mod validated_model_intrinsics; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs index 921cffd8fb..f90dcd8c3f 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs @@ -1,9 +1,6 @@ //! `_transform_cursor_intrinsics`: replace every intrinsic call, or raise its error. -use crate::cursor_intrinsics::_helpers::scan::{ - CallScan, ScanStep, call_scan, contains_intrinsic_name, step, -}; -use crate::sql_lexing::main::unclosed_message::unclosed_message; +use crate::cursor_intrinsics::_helpers::replacement::replaced_with_spans; /// `sql` with each cursor intrinsic call replaced and whether any was, or the malformed-call error. pub fn replace_intrinsics( @@ -11,45 +8,6 @@ pub fn replace_intrinsics( context: &str, replacement: impl Fn(&str) -> String, ) -> Result<(String, bool), String> { - if !contains_intrinsic_name(sql) { - return Ok((sql.to_owned(), false)); - } - let mut replaced: String = String::with_capacity(sql.len()); - let mut last: usize = 0; - let mut found: bool = false; - let mut index: usize = 0; - while index < sql.len() { - let (name, end): (&str, usize) = match step(sql, index) { - ScanStep::Next(next) => { - index = next; - continue; - } - ScanStep::Unclosed(construct) => return Err(unclosed_message(context, construct)), - ScanStep::Intrinsic { name, end } => (name, end), - }; - let close: usize = match call_scan(sql, end) { - CallScan::Called(close) => close, - CallScan::NotCalled => { - return Err(format!("{context} intrinsic {name} must be called with ()")); - } - CallScan::Arguments => { - return Err(format!( - "{context} intrinsic {name} does not accept arguments" - )); - } - CallScan::Unclosed(construct) => { - return Err(unclosed_message( - &format!("{context} cursor intrinsic"), - construct, - )); - } - }; - replaced.push_str(&sql[last..index]); - replaced.push_str(&replacement(name)); - last = close + 1; - index = close + 1; - found = true; - } - replaced.push_str(&sql[last..]); - Ok((replaced, found)) + replaced_with_spans(sql, context, replacement) + .map(|(replaced, spans)| (replaced, !spans.is_empty())) } diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs new file mode 100644 index 0000000000..002783e8a1 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs @@ -0,0 +1,13 @@ +//! Cursor intrinsic replacement that reports where each call moved. + +use crate::cursor_intrinsics::_helpers::replacement::replaced_with_spans; +use crate::cursor_intrinsics::models::IntrinsicSpan; + +/// `sql` with each cursor intrinsic call replaced and each call's byte span, or the call's error. +pub fn replace_intrinsics_with_spans( + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, Vec), String> { + replaced_with_spans(sql, context, replacement) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs index b358a56c80..c6301185ac 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs @@ -1,5 +1,6 @@ //! `get_validated_model_cursor_intrinsics`: check and canonicalize one model query's intrinsics. +use crate::cursor_intrinsics::_helpers::reserved::reserved_function_error; use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::cursor_intrinsics::models::IntrinsicModel; @@ -18,6 +19,9 @@ pub fn validated_model_intrinsics( "{context} contains a reserved internal cursor marker" )); } + if let Some(message) = reserved_function_error(sql, &context) { + return Err(message); + } let (canonical, found) = replace_intrinsics(sql, &context, |name| format!("{name}()"))?; if !found { return Ok(canonical); diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs index e6e5a52eb4..f01181f853 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs index 7474639f96..298b13f849 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs @@ -14,3 +14,12 @@ pub struct IntrinsicModel<'model> { pub incremental: bool, pub cursor: Option<&'model str>, } + +/// Where one replaced intrinsic call was in the source and is in the output, in bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct IntrinsicSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs index 3fb9bd9b42..c09ecfa1ac 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs @@ -90,6 +90,13 @@ fn given_sql_when_checking_cursor_intrinsics_then_python_acceptance_or_error_is_ sql: "SELECT ·__cursor_end", expected_check: rejected("Audit 'fresh' intrinsic __cursor_end must be called with ()"), }, + IntrinsicCheckTestCase { + description: "the reserved analysis function is rejected in any case", + sql: "SELECT __Polyglot_Typed(amount, 'BIGINT') FROM orders", + expected_check: rejected( + "Audit 'fresh' uses __polyglot_typed, a name reserved for SQL analysis; rename it", + ), + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-project/src/models.rs b/crates/sqlbuild-project/src/models.rs index 1c752d55df..3d6e8c124f 100644 --- a/crates/sqlbuild-project/src/models.rs +++ b/crates/sqlbuild-project/src/models.rs @@ -7,7 +7,7 @@ use sqlbuild_sqltext::compiler::models::CharSpan; use sqlbuild_analysis::assembly::analysis_session::models::{ContractProof, LineageFacts}; use sqlbuild_analysis::binding_locations::models::LocatedDiagnostic; use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_analysis::sql_macros::models::{SqlMacroCheck, TypedCall}; use sqlbuild_core::json::models::JsonValue; @@ -80,6 +80,8 @@ pub struct ModelInputRecord { pub expansion: ExpansionRecord, /// What the analysed SQL must show for the model's SQL macro call arguments to fit. pub sql_macro_checks: Vec, + /// Each expression macro call in `query_sql`, by byte range. + pub typed_calls: Vec, /// What assembly and the semantic checks finished, once the model is assembled. pub compiled: Option>, pub host: H::ModelInput, diff --git a/crates/sqlbuild-project/src/tests/helpers.rs b/crates/sqlbuild-project/src/tests/helpers.rs index e7bff79239..dfb8ee6c7e 100644 --- a/crates/sqlbuild-project/src/tests/helpers.rs +++ b/crates/sqlbuild-project/src/tests/helpers.rs @@ -48,6 +48,7 @@ pub(super) fn model_input(name: &str) -> ModelInputRecord { sql_validation_enabled: true, macro_deps: Vec::new(), sql_macro_checks: Vec::new(), + typed_calls: Vec::new(), expansion: ExpansionRecord { authored_sql: "SELECT 1".into(), expanded_sql: "SELECT 1".into(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 137e9a712e..9d387684be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; use crate::bindings::_helpers::compiled_views::model_input_view::{ - AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, + AttachedSchema, NativeModelInput, attached_schema, input_analysis_sql, input_model_name, input_reference_rows, input_sql_macro_checks, }; use crate::bindings::_helpers::native_host::host; @@ -132,7 +132,11 @@ fn session_request<'py>( complete_schemas: shapes.complete, catalog_schemas: shapes.schemas, dynamic_families_by_table: families, - models: model_rows(profile.py(), &request.model_inputs)?, + models: model_rows( + profile.py(), + &request.model_inputs, + &profile_dialect(profile)?, + )?, }) } @@ -323,15 +327,22 @@ fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<( } /// Each model's native request row, in analysis order. -fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyResult> { +fn model_rows<'py>( + py: Python<'py>, + model_inputs: &[Bound<'py, PyAny>], + dialect: &str, +) -> PyResult> { let enforced: Bound<'py, PyAny> = enum_member(py, RecordId::ContractPolicy, "ENFORCED")?; let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; let mut rows: Vec = Vec::with_capacity(model_inputs.len()); for model_input in model_inputs { let values: Bound<'py, PyAny> = native_field(model_input, "config")?.getattr("values")?; let get = |key: &str| mapping_get(&values, key); - let query_sql: String = - analysis_cursor_sql(py, &input_query_sql(model_input)?, &get("cursor_type")?)?; + let query_sql: String = analysis_cursor_sql( + py, + &input_analysis_sql(model_input, dialect)?, + &get("cursor_type")?, + )?; let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 22da623157..c2487bf4a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -186,6 +186,8 @@ pub(crate) struct NativeModelAnalysisSession { cache_path: Option, cache_stats: Option, rule_failure: RuleFailure, + /// The analysis dialect the session's models were marked and analysed in. + dialect: String, } /// What an internal native failure of the session names. @@ -208,6 +210,10 @@ impl NativeModelAnalysisSession { } /// The running session, or `NativeCompilerError` once it has failed or finished. + pub(crate) fn dialect(&self) -> &str { + &self.dialect + } + fn running(&mut self) -> PyResult { self.inner .take() @@ -347,6 +353,7 @@ pub(crate) fn start_model_analysis_session( request, ))?; let rule_failure: RuleFailure = RuleFailure::default(); + let dialect: String = request.dialect.clone(); request.nullability_callback = adapter_rules .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); let catalog = &catalog.inner; @@ -381,6 +388,7 @@ pub(crate) fn start_model_analysis_session( cache_path, cache_stats, rule_failure, + dialect, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 5eb41c050a..a5b9e0a289 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; +use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; @@ -14,6 +15,7 @@ use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_views::model_input_view::input_analysis_sql; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::project_assembly::syntax_validation::{ @@ -80,7 +82,7 @@ fn compiled_model<'py>( py, &analysis_cursor_sql( py, - &native_field(model_input, "query_sql")?.extract::()?, + &input_analysis_sql(model_input, &profile_dialect(&context.profile)?)?, &mapping_get(&config_values, "cursor_type")?, )?, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 5ef5b8dec2..2a7f6664d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -586,6 +586,7 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult PyResult> { +fn located( + input: &ModelInputRecord, + outcome: &ModelOutcome, + dialect: &str, +) -> PyResult> { let rows: &[DiagnosticRow] = &outcome.analysis.binding_diagnostics; if rows.is_empty() { return Ok(Vec::new()); @@ -165,7 +172,13 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> = vec![ + let mut typed_text: Option = None; + let typed: Option<(String, Vec)> = if *expansion.expanded_sql == *input.query_sql { + typed_analysis_pass(&expansion.expanded_sql, &input.typed_calls, dialect) + } else { + None + }; + let mut passes: Vec> = vec![ expansion .declaration_spans .iter() @@ -179,15 +192,28 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult { + passes.push(pass); + typed_text.insert(typed_sql) + } + None => &expansion.expanded_sql, + }; let source: BindingSource<'_> = BindingSource { authored, query, cleaned: &cleaned, - expansion: expansion - .authored - .then_some((&*expansion.expanded_sql, passes.as_slice())), + expansion: expansion.authored.then_some((expanded, passes.as_slice())), }; - locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err) + let mut located: Vec = + locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err)?; + for diagnostic in &mut located { + diagnostic.message = marker_free_text(&diagnostic.message).into_owned(); + if let Some(help) = diagnostic.help.as_mut() { + *help = marker_free_text(help).into_owned(); + } + } + Ok(located) } /// The SQL macro findings, then the binding diagnostics without unknown columns inside a call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index ee5260e7a5..2c801e2ef9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -9,6 +9,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; @@ -548,26 +549,29 @@ pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { +/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. +pub(crate) fn input_sql_macro_checks( + model_input: &Bound<'_, PyAny>, +) -> PyResult> { if let Ok(native) = model_input.cast::() { return native.get().read(model_input.py(), |record, _| { - Ok(record.query_sql.to_string()) + Ok(record.sql_macro_checks.clone()) }); } - native_field(model_input, "query_sql")?.extract() + Ok(Vec::new()) } -/// What the analysed SQL of a native model input must show for its SQL macro calls to fit. -pub(crate) fn input_sql_macro_checks( +/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. +pub(crate) fn input_analysis_sql( model_input: &Bound<'_, PyAny>, -) -> PyResult> { + dialect: &str, +) -> PyResult { if let Ok(native) = model_input.cast::() { return native.get().read(model_input.py(), |record, _| { - Ok(record.sql_macro_checks.clone()) + Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) }); } - Ok(Vec::new()) + native_field(model_input, "query_sql")?.extract() } /// Each reference of `model_input` as `(kind, name, package, call argument count)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index d3d532a8c0..b7ef9cdd1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::contracts::models::{ PromotionConflict, PromotionModel, PromotionRequest, }; +use sqlbuild_analysis::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use sqlbuild_analysis::type_system::errors::TypeNormalizationError; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -80,6 +81,7 @@ fn fact_contract_model(model: &CompiledModelFacts) -> ContractModel { unchecked_output_columns: analysis .map(|analysis| analysis.unchecked_output_columns.clone()) .unwrap_or_default(), + untyped_call_outputs: untyped_call_outputs(&model.query_sql, &model.typed_calls), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 149b0ea05d..447cc98dbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -14,6 +14,10 @@ use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge use sqlbuild_analysis::compiled_project::models::{ AnalysisGate, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, }; +use sqlbuild_analysis::sql_macros::main::moved_typed_calls::moved_typed_calls; +use sqlbuild_analysis::sql_macros::main::spliced_typed_calls::spliced_typed_calls; +use sqlbuild_analysis::sql_macros::models::{TextSplice, TypedCall}; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; @@ -681,6 +685,11 @@ impl NativeModelLoop { let query_sql: String = self .canonical_query(name, ¯o_sql, &values)? .map_err(|message| compile_error(context.classes, &message, None))?; + let typed_calls: Vec = query_typed_calls( + std::mem::take(&mut declarations.typed_calls), + (&declarations.spliced.sql, ¯o_sql, &query_sql), + ¯o_spans, + ); let opt_out_model: OptOutModel<'_, 'py> = OptOutModel { name, file_path, @@ -784,6 +793,7 @@ impl NativeModelLoop { analysis, named_usages, macros: macros.map(|macros| macros.output), + typed_calls, }) } @@ -973,6 +983,41 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::() } +/// The spliced SQL's expression macro calls moved through macro expansion into the query's bytes. +fn query_typed_calls( + calls: Vec, + (spliced, expanded, query): (&str, &str, &str), + macro_spans: &[SpanRow], +) -> Vec { + if calls.is_empty() { + return calls; + } + let calls: Vec = if macro_spans.is_empty() { + calls + } else { + spliced_typed_calls(calls, (spliced, expanded), &spliced_spans(macro_spans)) + }; + if expanded == query { + return calls; + } + let Ok((canonical, spans)) = + replace_intrinsics_with_spans(expanded, "", |name| format!("{name}()")) + else { + return Vec::new(); + }; + if canonical != query { + return Vec::new(); + } + let splices: Vec = spans + .into_iter() + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) + .collect(); + moved_typed_calls(calls, &splices) +} + /// Span rows as the spliced spans the scan-fault locator maps through. fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index c5b50f0881..deaf891fc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -9,6 +9,7 @@ use sqlbuild_analysis::compiled_project::main::plain_model_facts::plain_model_fa use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, ModelConfigFacts, PlainModel, }; +use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_discovery::model_files::models::HeaderFacts; use crate::bindings::_helpers::compiled_project::project::{ @@ -100,6 +101,8 @@ pub(crate) struct FinishedModel<'py> { pub(crate) named_usages: Vec>, /// The query's macro expansion, when it calls macros. pub(crate) macros: Option, + /// Each expression macro call in the query, by byte range. + pub(crate) typed_calls: Vec, } /// The Python classes a finished model's record is built with. @@ -235,6 +238,7 @@ pub(crate) fn finished_record<'py>( sql_validation_enabled: finished.analysis.0, macro_deps, sql_macro_checks: finished.declarations.sql_macro_checks.clone(), + typed_calls: finished.typed_calls.clone(), expansion: ExpansionRecord { authored: *interpolated_sql == *authored_query_sql, authored_sql: authored, @@ -286,6 +290,7 @@ pub(crate) fn finished_record<'py>( enum_columns, enum_declarations: declarations(enums)?, constant_declarations: declarations(constants)?, + typed_calls: finished.typed_calls, }); Ok((record, facts)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 7f3f2135aa..5428788f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -11,7 +11,7 @@ use sqlbuild_analysis::compiled_project::main::splice_declarations::splice_decla use sqlbuild_analysis::compiled_project::models::{ DeclarationSplice, InaccessibleDeclaration, SplicedSql, }; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_analysis::sql_macros::models::{ExpandedText, SqlMacroCheck, TypedCall}; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::sql_macro_host::{ @@ -82,6 +82,8 @@ pub(crate) struct ExpandedDeclarations<'py> { pub(crate) sql_macros: Vec, /// What the analysed SQL must show for each top-level call's arguments to fit. pub(crate) sql_macro_checks: Vec, + /// Each expression macro call in the spliced SQL, by byte range. + pub(crate) typed_calls: Vec, } /// Splice each reference in order, raising the attachment's error for the first bad one. @@ -98,12 +100,17 @@ pub(crate) fn expand_declarations<'py>( .map(|reference| reference.start) .collect(); let (mut call_texts, sql_macros, sql_macro_checks) = if call_starts.is_empty() { - (Vec::new().into_iter(), Vec::new(), Vec::new()) + ( + Vec::::new().into_iter(), + Vec::new(), + Vec::new(), + ) } else { let (texts, facts) = expanded_sql_macro_calls(request, &call_starts)?; usages.extend(facts.usages); (texts.into_iter(), facts.names, facts.checks) }; + let mut call_splices: Vec<(usize, Vec)> = Vec::new(); for ScannedReference { kind, name, @@ -113,10 +120,14 @@ pub(crate) fn expand_declarations<'py>( } in request.references { if is_sql_macro_reference(*kind) { + let expanded: ExpandedText = call_texts.next().unwrap_or_default(); + if !expanded.calls.is_empty() { + call_splices.push((splices.len(), expanded.calls)); + } splices.push(DeclarationSplice { start: *start, end: *end, - replacement: call_texts.next().unwrap_or_default(), + replacement: expanded.sql, }); continue; } @@ -144,14 +155,54 @@ pub(crate) fn expand_declarations<'py>( if let Some(code) = request.stop { return Err(compile_error(request, &DeclarationError::ScanStop(code))); } + let spliced: SplicedSql = splice_declarations(request.sql, &splices); + let typed_calls: Vec = spliced_call_positions(&spliced, call_splices); Ok(ExpandedDeclarations { - spliced: splice_declarations(request.sql, &splices), + spliced, usages, sql_macros, sql_macro_checks, + typed_calls, }) } +/// Each call text's typed calls moved to where its splice landed in `spliced`, by byte. +fn spliced_call_positions( + spliced: &SplicedSql, + call_splices: Vec<(usize, Vec)>, +) -> Vec { + let mut typed: Vec = Vec::new(); + if call_splices.is_empty() { + return typed; + } + let ascii: bool = spliced.sql.is_ascii(); + let mut characters = spliced.sql.char_indices().map(|(offset, _)| offset); + let mut seen: usize = 0; + let mut last: usize = 0; + for (index, calls) in call_splices { + let Some(span) = spliced.spans.get(index) else { + continue; + }; + let offset: usize = if ascii { + span.output_start + } else { + if span.output_start >= seen { + last = characters + .nth(span.output_start - seen) + .unwrap_or(spliced.sql.len()); + seen = span.output_start + 1; + } + last + }; + typed.extend(calls.into_iter().map(|call| TypedCall { + start: call.start + offset, + end: call.end + offset, + ..call + })); + } + typed +} + fn enum_text<'py>( request: &DeclarationExpansion<'_, 'py>, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index b57d8924e1..bbeb5bc735 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -8,18 +8,20 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::sql_macros::constants::{ - BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, MIXED_KIND_CODE, + BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, BODY_TYPE_CODE, MIXED_KIND_CODE, RETURNS_MISMATCH_CODE, }; use sqlbuild_analysis::sql_macros::errors::ExpansionFailure; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; use sqlbuild_analysis::sql_macros::main::bare_macro_calls::bare_macro_calls; +use sqlbuild_analysis::sql_macros::main::byte_text_splices::byte_text_splices; use sqlbuild_analysis::sql_macros::main::declaration_visible_from_macro::declaration_visible_from_macro; use sqlbuild_analysis::sql_macros::main::expand_sql_macro_call::expand_sql_macro_call; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; use sqlbuild_analysis::sql_macros::main::render_inline_collection::render_inline_collection; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; use sqlbuild_analysis::sql_macros::models::{ - InsertedNames, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroLocation, TextContext, + ExpandedText, InsertedNames, RenderedDeclarations, SqlMacroCall, SqlMacroCheck, + SqlMacroDefinition, SqlMacroLocation, TextContext, TextSplice, }; use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -94,11 +96,11 @@ pub(crate) fn is_sql_macro_reference(kind: u8) -> bool { kind == SQL_MACRO_REFERENCE_KIND } -/// The text replacing each SQL macro call starting at the given character offsets of `request.sql`. +/// The replacement text, with its expression macro calls, of each SQL macro call at `starts`. pub(crate) fn expanded_sql_macro_calls<'py>( request: &DeclarationExpansion<'_, 'py>, starts: &[usize], -) -> PyResult<(Vec, SqlMacroFacts<'py>)> { +) -> PyResult<(Vec, SqlMacroFacts<'py>)> { let sql: &str = request.sql; let calls: Vec = sql_macro_calls(sql).map_err(|error| macro_error(request, &error))?; @@ -125,7 +127,7 @@ pub(crate) fn expanded_sql_macro_calls<'py>( owner: &owner, }; let mut inserted: InsertedNames = InsertedNames::default(); - let mut replacements: Vec = Vec::with_capacity(starts.len()); + let mut replacements: Vec = Vec::with_capacity(starts.len()); for start in starts { let Some(call) = by_char.get(start) else { return Err(macro_error( @@ -146,6 +148,7 @@ pub(crate) fn expanded_sql_macro_calls<'py>( Err(ExpansionFailure::Host(error)) => return Err(error), Err(ExpansionFailure::Macro(error)) => { let located: SqlMacroError = match host.call_location((call.start, call.end)) { + Some(location) if is_macro_file_error(&error) => error.at(location), Some(location) => error.at_call(location), None => error, }; @@ -471,10 +474,13 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { &self, scope: &SqlMacroScope, text: &str, - ) -> Result> { + ) -> Result> { let (references, stop) = declaration_scan(text); if references.is_empty() && stop.is_none() { - return Ok(text.to_owned()); + return Ok(RenderedDeclarations { + sql: text.to_owned(), + splices: Vec::new(), + }); } let context: Bound<'py, PyAny> = host(self.scope_context(scope))?; if let SqlMacroScope::Macro(definition) = scope { @@ -548,7 +554,12 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { sql_macros: None, }))?; self.facts.borrow_mut().usages.extend(expanded.usages); - Ok(expanded.spliced.sql) + let splices: Vec = + byte_text_splices(text, &expanded.spliced.sql, &expanded.spliced.spans); + Ok(RenderedDeclarations { + sql: expanded.spliced.sql, + splices, + }) } fn constant_shape( @@ -620,3 +631,8 @@ fn declaration_path( } Ok(None) } + +/// Whether `error` reports the macro's own body rather than one call of it. +fn is_macro_file_error(error: &SqlMacroError) -> bool { + [RETURNS_MISMATCH_CODE, BODY_TYPE_CODE].contains(&error.code) +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 9fedfc925e..afe6dbf8d4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -50,6 +50,7 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result .iter() .map(declaration) .collect(), + typed_calls: facts.typed_calls.clone(), }) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index 15c69a0ea5..f0b485612c 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -132,6 +132,7 @@ impl RowWriter { relative_path, query_sql: _, authored_sql: _, + typed_calls: _, config, authored_config_keys, logical_schema, @@ -348,6 +349,7 @@ impl RowReader<'_> { relative_path: self.text()?, query_sql: String::new(), authored_sql: String::new(), + typed_calls: Vec::new(), config: { let count: usize = self.count()?; let mut config: BTreeMap = BTreeMap::new(); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index eda014fbd7..4a94856b59 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -94,6 +94,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { enum_columns: vec!["status".to_owned()], enum_declarations: vec![declaration.clone()], constant_declarations: vec![declaration], + typed_calls: Vec::new(), declared_audit_count: 3, targeting_test_count: 4, empty_input_only_test_count: 0, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index d8a6b21248..f045e25276 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use sqlbuild_analysis::sql_macros::models::TypedCall; use crate::errors::RowsError; use serde_json::Value; @@ -569,6 +570,9 @@ pub struct Model { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + #[serde(default)] + pub typed_calls: Vec, pub declared_audit_count: u32, pub targeting_test_count: u32, #[serde(skip_deserializing)] @@ -877,6 +881,8 @@ pub struct ModelRow { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A model's declared schema entry. diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index fe32b68f45..5af9afca09 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -322,6 +322,15 @@ impl Builder<'_> { direct_projection: column_name(value).is_some(), direct_column: column_name(unwrapped), cast_type, + span: range + .as_ref() + .filter(|range| !range.is_empty()) + .map(|range| { + ( + self.tokens.start(range.start), + self.tokens.end(range.end - 1), + ) + }), start: range.and_then(|range| self.tokens.leaf_start(&range)), }) } diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index 8f77a5c64b..db8a95d7d2 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -89,6 +89,8 @@ pub(crate) struct ExpressionItem { pub(crate) direct_column: Option, pub(crate) cast_type: Option, pub(crate) start: Option, + /// The normalized characters from the expression's first token to its last, alias excluded. + pub(crate) span: Option<(usize, usize)>, } #[derive(Debug)] @@ -251,6 +253,14 @@ impl<'a> ModelSource<'a> { .unwrap_or_else(|| self.bytes.len() - 1) } + /// The model's byte range from normalized character `start` to the character before `end`. + pub(crate) fn byte_range(&self, start: usize, end: usize) -> (usize, usize) { + let last: usize = self.bytes.len() - 1; + let first: usize = self.bytes[self.original(start).min(last)]; + let after: usize = self.bytes[(self.original(end.saturating_sub(1)) + 1).min(last)]; + (first, after.max(first)) + } + /// The 1-based line and column of normalized character `offset`. pub(crate) fn line_column(&self, offset: usize) -> (u64, u64) { let character = self.original(offset); diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs index d760285923..74c57fecbc 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs @@ -1,10 +1,8 @@ -use std::cell::OnceCell; - -use sqlbuild_analysis::sql_macros::main::sql_macro_output_names::sql_macro_output_names; +use sqlbuild_analysis::sql_macros::models::TypedCall; use crate::constants::ENFORCED_CONTRACT; use crate::models::{Column, Fault, Model, RuleMetadata}; -use crate::query_facts::models::{ItemFacts, QueryFacts, SelectFacts, SetFacts}; +use crate::query_facts::models::{ExpressionItem, ItemFacts, QueryFacts, SelectFacts, SetFacts}; use crate::rules::_helpers::evaluation::{ ParsedModel, plain_select, reads_dependency, sole_table_name, }; @@ -15,10 +13,6 @@ struct OutputScope<'a> { columns: &'a [Column], set_branch: bool, by_name: bool, - /// Where SQL macro call items are read: the main query (`Some(None)`), a CTE, or nowhere. - macro_scope: Option>, - /// The first select's branch of the scope's set operation, left to right. - branch: usize, } struct OutputRule<'a> { @@ -26,8 +20,6 @@ struct OutputRule<'a> { model: &'a Model, rule: &'a RuleMetadata, faults: &'a FaultCollector, - /// The main query's outputs that are one SQL macro call each, read when first needed. - sql_macro_outputs: OnceCell, usize, String)>>, } pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { @@ -54,18 +46,14 @@ pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &F model, rule, faults, - sql_macro_outputs: OnceCell::new(), }; if let Some((boundary, columns)) = final_cte_output_boundary(query, &model.columns) { - let final_cte: Option<&str> = query.ctes.last().map(|cte| cte.name.as_str()); evaluator.evaluate_set( &boundary.body, OutputScope { columns: &columns, set_branch: false, by_name: false, - macro_scope: final_cte.map(Some), - branch: 0, }, ); return; @@ -77,8 +65,6 @@ pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &F columns: &columns, set_branch: false, by_name: false, - macro_scope: Some(None), - branch: 0, }, ); } @@ -142,17 +128,9 @@ impl OutputRule<'_> { columns: scope.columns, set_branch: true, by_name: *by_name, - macro_scope: scope.macro_scope, - branch: scope.branch, }; self.evaluate_set(left, branch_scope); - self.evaluate_set( - right, - OutputScope { - branch: scope.branch + set_branches(left), - ..branch_scope - }, - ); + self.evaluate_set(right, branch_scope); } SetFacts::Other => self.push( None, @@ -209,11 +187,12 @@ impl OutputRule<'_> { .cast_type .as_ref() .is_some_and(|data_type| type_spelling_equal(data_type, &column.data_type)); + let call: Option<&TypedCall> = self.whole_call(expression); let passes = cast_matches_contract || column.type_proven && direct && !scope.set_branch || column.type_proven - && !explicit_cast - && self.sql_macro_output(scope, output_name.as_deref()); + && call + .is_some_and(|call| call.returns.is_some() || call.returns_like.is_some()); if passes { continue; } @@ -233,31 +212,30 @@ impl OutputRule<'_> { "output {:?} {reason}; declared type is {}", name, column.data_type ), - format!( - "Wrap the complete expression in CAST(... AS {}) or establish that exact type on a direct upstream column.", - column.data_type - ), + match call.filter(|call| call.returns.is_none() && call.returns_like.is_none()) { + Some(call) => format!( + "Declare returns {} in SQL macro {}(), or wrap the call in CAST(... AS {}).", + column.data_type, call.macro_name, column.data_type + ), + None => format!( + "Wrap the complete expression in CAST(... AS {}) or establish that exact type on a direct upstream column.", + column.data_type + ), + }, ); } } - /// Whether output `name` in `scope`'s select is one SQL macro call, typed by its `returns`. - fn sql_macro_output(&self, scope: OutputScope<'_>, name: Option<&str>) -> bool { - let (Some(macro_scope), Some(name)) = (scope.macro_scope, name) else { - return false; - }; - self.sql_macro_outputs - .get_or_init(|| sql_macro_output_names(&self.model.authored_sql)) - .iter() - .any(|(cte, branch, output)| { - output.eq_ignore_ascii_case(name) - && *branch == scope.branch - && match (cte.as_deref(), macro_scope) { - (None, None) => true, - (Some(cte), Some(scope)) => cte.eq_ignore_ascii_case(scope), - _ => false, - } - }) + /// The expression SQL macro call `expression` is the whole of, found by its byte range. + fn whole_call(&self, expression: &ExpressionItem) -> Option<&TypedCall> { + let (start, end) = expression.span?; + let (start, end) = self.parsed.source.byte_range(start, end); + let sql: &str = &self.model.query_sql; + self.model.typed_calls.iter().rev().find(|call| { + call.start == start + && end <= call.end + && sql.get(end..call.end).is_some_and(only_trivia) + }) } fn push(&self, offset: Option, message: String, remediation: String) { @@ -274,11 +252,23 @@ impl OutputRule<'_> { } } -/// The selects of `set_expr`'s set operation, a nested query counting as one. -fn set_branches(set_expr: &SetFacts) -> usize { - match set_expr { - SetFacts::SetOperation { left, right, .. } => set_branches(left) + set_branches(right), - SetFacts::Select(_) | SetFacts::Query(_) | SetFacts::Other => 1, +/// Whether `text` holds only whitespace and comments. +fn only_trivia(mut text: &str) -> bool { + loop { + text = text.trim_start(); + if text.is_empty() { + return true; + } + if let Some(comment) = text.strip_prefix("--") { + text = comment.split_once('\n').map_or("", |(_, rest)| rest); + } else if let Some(comment) = text.strip_prefix("/*") { + let Some((_, rest)) = comment.split_once("*/") else { + return false; + }; + text = rest; + } else { + return false; + } } } diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs index a1af8326ea..f99ce7bcaa 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs @@ -295,10 +295,10 @@ pub(super) fn cte_macro_indices(tokens: &[Token]) -> HashSet { } else if token.text.starts_with(LINT_SENTINEL_PREFIX) && scopes.contains(&depths[index]) && index > 0 - && matches!( + && (matches!( tokens[index - 1].token_type, TokenType::With | TokenType::Comma - ) + ) || indices.contains(&(index - 1))) && tokens.get(index + 1).is_some_and(|next| { matches!(next.token_type, TokenType::Comma | TokenType::Select) || starts_cte(tokens, index + 1) diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs index b5d068a929..b748af51a2 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs @@ -169,6 +169,13 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "WITH\n__sqb_lint_0__,\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", }, + test_types::LayoutRuleTestCase { + description: "comma-less CTE macro placeholders in a row are separated like CTEs", + dialect: "duckdb", + sql: "WITH a AS (SELECT 1 AS x), __sqb_lint_0__ __sqb_lint_1__ paid AS (SELECT * FROM a) SELECT * FROM paid", + line_width: 100, + expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\n__sqb_lint_0__\n\n__sqb_lint_1__\npaid AS (\n SELECT *\n FROM a\n)\nSELECT *\nFROM paid", + }, test_types::LayoutRuleTestCase { description: "an alias and comma that push a call over the width break the call", dialect: "snowflake", diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs index 0de0113fd9..eb5d8398a2 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs @@ -76,11 +76,14 @@ fn orders_by_all(select: &Select) -> bool { .order_by .iter() .flat_map(|order| order.expressions.iter()) - .any(|ordered| { - matches!(&ordered.this, Expression::Column(column) - if column.table.is_none() + .any(|ordered| match &ordered.this { + Expression::Column(column) => { + column.table.is_none() && !column.name.quoted - && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL)) + && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL) + } + Expression::Var(var) => var.this.eq_ignore_ascii_case(ORDER_BY_ALL), + _ => false, }) } diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index c030d8865e..7aa0034670 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -683,10 +683,12 @@ def _sql_macro_cases() -> tuple[FailureCase, ...]: failure_case( name="sql-macro-invalid-definition", expected_code="P015", - expected_message="expression macro x() needs returns in its MACRO header", + expected_message="returns TYPE OF w names no argument of macro x()", files=_sql_macro_columns( columns="@sql.x(amount) AS amount, status", - files=_x_macro(arguments="v DOUBLE", returns="", body="@v + 1\n"), + files=_x_macro( + arguments="v DOUBLE", returns=", returns TYPE OF w", body="@v + 1\n" + ), ), ), failure_case( @@ -786,6 +788,26 @@ def _sql_macro_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="sql-macro-returns-disagrees-with-body", + expected_code="P024", + expected_message="macro x() declares returns DOUBLE, but its body is VARCHAR", + files=_sql_macro_columns( + columns="@sql.x(amount) AS amount, status", + files=_x_macro( + arguments="v DOUBLE", returns=", returns DOUBLE", body="CAST(@v AS VARCHAR)\n" + ), + ), + ), + failure_case( + name="sql-macro-body-misuses-an-any-family", + expected_code="P025", + expected_message="body of macro x() does not accept every value of v ANY NUMERIC", + files=_sql_macro_columns( + columns="@sql.x(amount) AS amount, status", + files=_x_macro(arguments="v ANY NUMERIC", returns="", body="UPPER(@v)\n"), + ), + ), failure_case( name="sql-macro-body-declaration-not-visible", expected_code="S026", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index af4b3b125a..94ccae6a9c 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -564,6 +564,8 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "P021", "P022", "P023", + "P024", + "P025", "S026", } ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index 82417bf04e..b00e5ad7b5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -13,7 +13,9 @@ Online: https://sqlbuild.com/docs/concepts/macros/sql-macros/ - Parameter types - Parameters in the body - Compose macros -- Checks and tools +- Checks +- Result types +- Tools A SQL macro is a reusable piece of SQL with typed parameters. Unlike a [Python macro](python-macros.md), it holds no logic: its body is SQL, so the compiler checks @@ -64,10 +66,11 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` -`returns` is required for expression macros and not allowed for the other forms. +`returns` is optional for expression macros and not allowed for the other forms. See +[Result types](#result-types) for what it does. The description follows the same rules as every header string: write text that contains double quotes as `"""..."""`, and a wrapped description is stored folded into one flowing text (see @@ -93,7 +96,7 @@ compiles to: ```sql SELECT o.order_id, - ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees + CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM analytics.stg_orders AS o ``` @@ -131,6 +134,28 @@ The compiled model holds `orders_raw`, `live`, `ranked`, `latest` and `final`, f order, and later CTEs may read any of the macro's CTEs. Commas around the call are optional; `sqb format` normalises them. +The last CTE of a CTE-list macro is its result. To call one macro more than once in a model, name +each call's CTEs with the reserved `__cte` argument: + +```sql +WITH eu_raw AS (SELECT * FROM __ref("stg_orders_eu")), +us_raw AS (SELECT * FROM __ref("stg_orders_us")), +@sql.clean_orders(eu_raw, __cte := eu_orders) +@sql.clean_orders(us_raw, __cte := us_orders) +final AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders) +SELECT * FROM final +``` + +The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so +this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, +`us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are +renamed with them, wherever a name reads the macro's CTE: a table alias such as `FROM live AS l`, +or a nested `WITH` that defines its own `live`, keeps its own meaning. Arguments are never renamed, +even one named like a macro CTE. Later CTEs may read the prefixed ones too. + +`__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in +`arguments`. A call without it keeps the macro's own names. + ### Column lists A column-list macro inserts its columns into the caller's select list. Every column needs a name: @@ -152,7 +177,8 @@ Use a column list when the same ordered set of columns repeats, for example in e `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. A CTE or column name a macro inserts that the caller already defines fails with `P018`, which names -both. Nothing is renamed, so a model cannot call the same CTE-list macro twice. +both. Calling the same CTE-list macro twice without `__cte` fails the same way, and the error +suggests `__cte`. ## Parameter types @@ -162,6 +188,7 @@ Every parameter has a type, and may have a `DEFAULT`: |---|---| | `BIGINT`, `VARCHAR`, `TIMESTAMP`, ... | Any SQL expression of a compatible type, evaluated per row: a column, arithmetic, a function call, a literal or a constant. | | `ANY` | Any scalar SQL expression, whatever its type: a column, a literal, an expression or a scalar constant. Its type is not checked. Lists and relations fail with `P017`. | +| `ANY NUMERIC`, `ANY TEXT`, `ANY TEMPORAL` | Like `ANY`, but only values of one family: integer, decimal and float types; string types; or date, time and timestamp types. | | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | @@ -179,6 +206,19 @@ still checked, and the argument's type counts as unknown there. `ANY` is not `VA value of any type. `ANY` takes a `DEFAULT` but not `CONST`, which needs a type to check values against, and it cannot be a `LIST` or `SET` element type. +### Choosing an argument type + +- A fixed type, such as `BIGINT`, when the body needs that type. Callers may pass any compatible + value, and the body is checked with that type. +- A family, such as `ANY NUMERIC`, when the body works for every type of the family, such as + `ROUND(@value, @places)`. A call whose argument has a known type outside the family fails with + `P017` at the argument. The body is checked as if the argument had the family's widest type + (`DOUBLE`, `VARCHAR` or `TIMESTAMP`), so `UPPER(@value)` on `ANY NUMERIC` fails with `P025` at the + macro file. Pair a family with `returns TYPE OF value` to return the caller's own type. +- `ANY` when the body works for values of any type, typically because it casts the argument itself. + +Families take a `DEFAULT` but not `CONST`, and are not `LIST` or `SET` element types. + Leave `CONST` off unless the body needs a literal. `CONST` is part of the macro's contract: callers can no longer pass a column. @@ -207,7 +247,7 @@ These fail with `P022`: - a relation parameter written bare where it would read a table, such as `FROM src` or `src.created_at`; the error says `src is a parameter of macro clean_orders(); write @src` -Parameters may not be named `sql`, `py`, `const`, `enum` or `param`. +Parameters may not be named `sql`, `py`, `const`, `enum`, `param` or `__cte`. ### Positions that need a literal @@ -256,7 +296,7 @@ isn't visible from there fails with `S026`; to use a caller's value, make it an `@const(...)` at the call site. SQL macros a body calls resolve the same way, so a macro can call a folder-local macro beside it that its callers cannot see. -## Checks and tools +## Checks Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: @@ -282,18 +322,48 @@ parameters of a calling macro, and calls in models with runtime placeholders; th column checks still run on the expanded SQL. All other errors also point at the call and at the macro file. -A select-list item that is one expression macro call under an alias, as in -`@sql.cents_to_dollars(o.amount_cents) AS amount`, has the macro's `returns` type wherever the -compiler infers no type for the expanded SQL, or exactly that type. This holds in the main query -and in any top-level CTE, including each branch of their `UNION`, `INTERSECT` or `EXCEPT`, and the -type flows on as an inferred type would: through `SELECT * FROM final`, pass-through columns and -joins, to contract checks (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A -set operation's column combines its branches as usual, so a sibling branch of another type still -leaves it unproven, and a CTE holding one is read by column name, since `SELECT *` over it is not -typed. Such a call needs no `CAST` to satisfy a matching contract column. Any other -inferred type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and `K002` -reports the mismatch. A call inside a larger expression, a subquery, a nested `WITH` or a set -operation with a parenthesised branch is typed from the expanded SQL as before. +## Result types + +An expression macro call is typed wherever it stands, from its expanded SQL, as if the SQL were +written there. `returns` declares the type for when that SQL's type is unknown: + +```sql +MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); + +bar(LENGTH(@value), 0, 10) +``` + +- The declared type applies only where the expanded SQL's type is unknown. A partly unknown type, + such as `[]`, an array of unknown elements, takes `returns` only when `returns` has its shape, + such as `BIGINT[]`. A known type stands, even a close one such as `VARCHAR(100)` for + `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. +- The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each + branch of a set operation, through operators and functions around the call, to contract checks + (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A select-list item that is + one call to a macro with `returns` needs no `CAST` for `SQBRCONTRACT105`; a call inside a larger + expression still does. +- Without `returns`, the call is typed by inference alone. When an output is one call whose type is + unknown, `K003` and `SQBRCONTRACT105` suggest declaring `returns` on the macro. +- `returns TYPE OF ` declares the type of that argument at each call, under the same + rule: `arguments (value ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF value`. If the + argument's own type is unknown, so is the call's. The argument must be a scalar argument of the + macro (`P015`). +- A fixed `returns` is checked against the body at the macro file. The body is typed from the + declared argument types; a known type of another family, such as `UPPER(@value)` with + `returns BIGINT`, fails with `P024`, as does a partly unknown body of another shape, such as `[]` + with `returns BIGINT`. A body whose type cannot be inferred, for example because it reads `ANY` + arguments or calls a function whose type is unknown or only guessed, trusts `returns`. +- `returns` must be a type the adapter's SQL dialect accepts, including a decimal precision and + scale it can hold, such as at most `DECIMAL(38, s)` on DuckDB (`P015`). +- A call where only a keyword can stand, such as the date part of `EXTRACT`, is typed by inference + alone. +- A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed + by inference alone. +- `returns` never changes the compiled SQL. + +A `* REPLACE` item takes its source column's type rather than the call's. + +## Tools - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 7e5146c01f..6ef7862a67 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -615,7 +615,9 @@ __macro_actual__ AS (SELECT order_id, updated_at FROM latest), __macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at) ``` -Other test modes may not call a CTE-list macro in their `WITH` list. +These calls may pass `__cte := name`, as in a model, to call the macro more than once or to give +its CTEs names of the test's own. Other test modes may not call a CTE-list macro in their `WITH` +list. ### UDF tests diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md index dc17d12f6f..52bb4c35de 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md @@ -28,7 +28,7 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` ```sql @@ -36,6 +36,12 @@ SELECT order_id, @sql.cents_to_dollars(amount_cents) AS amount_dollars FROM __ref("stg_orders") ``` +`returns` is optional: a call is typed from its expanded SQL, and `returns` (or +`returns TYPE OF `) applies only where that type is unknown. A `returns` whose body has a +known type of another family fails with `P024`. Narrow `ANY` arguments with `ANY NUMERIC`, +`ANY TEXT` or `ANY TEMPORAL`, and call a CTE-list macro more than once in a model with +`__cte := name`. + Full reference: [docs/concepts/macros/sql-macros.md](docs/concepts/macros/sql-macros.md). **Python macros** live in `macros/python/` and are plain Python functions that return final SQL diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py index 147ec3b2f3..06b81ef83a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py @@ -80,3 +80,26 @@ class SqlMacroCompileTestCase: expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = () inspected_paths: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SqlMacroLineageTestCase: + """A column read through a SQL macro call, and the same column written inline.""" + + description: str + macro_files: dict[str, str] + inline_files: dict[str, str] + column: str + expected_edge: str + + +@dataclass(frozen=True) +class SqlMacroWarmCompileTestCase: + """A compiled project whose macro file is edited, then compiled warm, and what each prints.""" + + description: str + files: dict[str, str] + edited_path: str + replaced_text: tuple[str, str] + expected_cold_fragment: str + expected_warm_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index 01e92ccaca..b071b39384 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re import subprocess from collections.abc import Mapping from pathlib import Path @@ -52,7 +53,7 @@ 'MACRO (\n description "Cents to dollars",\n' " arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n" " returns DECIMAL(18, 2),\n);\n\n" - "ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places)\n" + "CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2))\n" ) } NET_AMOUNT: dict[str, str] = { @@ -232,13 +233,25 @@ def sql_text_macros(folder: str, names: list[str]) -> dict[str, str]: } -def text_macro(name: str, returns: str, body: str) -> dict[str, str]: - """The SQL macro `name` taking one VARCHAR value and declaring `returns`.""" +def text_macro(name: str, returns: str, body: str, argument: str = "VARCHAR") -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value and declaring `returns`.""" + + return _value_macro(name, f" returns {returns},\n", body, argument) + + +def untyped_text_macro(name: str, body: str, argument: str = "VARCHAR") -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value without declaring `returns`.""" + + return _value_macro(name, "", body, argument) + + +def _value_macro(name: str, declared: str, body: str, argument: str) -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value, with its `declared` header lines.""" return { f"macros/sql/{name}.sql": ( - f'MACRO (\n description "Text from a value",\n arguments (value VARCHAR),\n' - f" returns {returns},\n);\n\n{body}\n" + f'MACRO (\n description "Text from a value",\n arguments (value {argument}),\n' + f"{declared});\n\n{body}\n" ) } @@ -252,3 +265,50 @@ def contract_model(name: str, columns: str, sql: str) -> dict[str, str]: f" columns ({columns}),\n);\n\n{sql}" ) } + + +def typed_position_files(shape: str, sql: str, declared: str) -> dict[str, str]: + """The `orders` model over `sql`, contracted itself, contracted downstream, or uncontracted.""" + + downstream: dict[str, str] = contract_model( + "order_x", f"x (type {declared})", 'SELECT b.x FROM __ref("orders") AS b\n' + ) + shapes: dict[str, dict[str, str]] = { + "contracted": contract_model("orders", f"x (type {declared})", sql), + "downstream": {**orders_model(sql), **downstream}, + "uncontracted": orders_model(sql), + } + return shapes[shape] + + +def lineage_edges(printed: str) -> list[str]: + """The `source -> target` edges a list-format lineage command printed.""" + + return [edge.strip() for edge in re.findall(r"^.*->.*$", printed, flags=re.MULTILINE)] + + +def adapter_project(adapter: str) -> dict[str, str]: + """The orders project configured for `adapter`, compiled without a warehouse.""" + + return { + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{adapter}"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" + ) + } + + +def picked_orders(body: str) -> dict[str, str]: + """The CTE-list macro `pick` over relation `src` with `body`, called once as `__cte := a`.""" + + return { + "macros/sql/pick.sql": ( + 'MACRO (\n description "Picked orders",\n' + " arguments (src RELATION (order_id BIGINT)),\n);\n\n" + body + "\n" + ), + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.pick(raw, __cte := a)\nSELECT * FROM a\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py index fea2ea77b5..7ae41d98ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py @@ -3,6 +3,7 @@ from __future__ import annotations import subprocess +from decimal import Decimal from pathlib import Path import pytest @@ -145,7 +146,7 @@ ), }, query="SELECT order_id, amount, fees, net, label FROM orders", - expected_rows=[(2, 5.0, 1.0, 4.5, "[5.0]")], + expected_rows=[(2, Decimal("5.00"), Decimal("1.00"), Decimal("4.50"), "[5.00]")], ), SqlMacroBuildTestCase( description="cte_list_and_column_list_macros_with_enum_and_list_constant_arguments", @@ -458,7 +459,7 @@ def test_given_any_spelling_of_list_calls_when_building_then_the_model_holds_the files={ **CENTS_TO_DOLLARS, "macros/sql/zz_invalid.sql": ( - 'MACRO (description "Invalid", arguments (v BIGINT));\n\n@v + 1\n' + 'MACRO (description "Invalid", returns BIGINT);\n\nWITH a AS (SELECT 1)\n' ), **orders_model( 'SELECT @sql.cents_to_dollars(amount_cents) AS d FROM __ref("stg_orders")\n' @@ -468,7 +469,7 @@ def test_given_any_spelling_of_list_calls_when_building_then_the_model_holds_the expected_returncode=1, expected_fragments=( "Resource: macro:cents_to_dollars", - "ERROR S021 macros/sql/zz_invalid.sql: expression macro zz_invalid() needs returns", + "ERROR S021 macros/sql/zz_invalid.sql: CTE-list macro zz_invalid() cannot declare returns", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py new file mode 100644 index 0000000000..e91aafcafe --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py @@ -0,0 +1,304 @@ +"""E2E coverage for the `__cte` argument naming the CTEs a CTE-list SQL macro call inserts.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroBuildTestCase, + SqlMacroCommandTestCase, + SqlMacroErrorTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + ALL_MACROS, + CLEAN_ORDERS, + DATABASE, + contract_model, + inspected_text, + orders_model, + output, + picked_orders, + run_project, + text_macro, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb + +_TWO_CALLS: str = ( + "WITH gb_raw AS (SELECT * FROM __ref(\"stg_orders\") WHERE country_code = 'GB'),\n" + "fr_raw AS (SELECT * FROM __ref(\"stg_orders\") WHERE country_code = 'FR'),\n" + "@sql.clean_orders(gb_raw, __cte := gb_orders)\n" + "@sql.clean_orders(fr_raw, __cte := fr_orders)\n" + "final AS (\n" + " SELECT order_id, amount_cents FROM gb_orders\n" + " UNION ALL SELECT order_id, amount_cents FROM fr_orders\n" + ")\nSELECT * FROM final\n" +) +_INTERMEDIATE: str = ( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.clean_orders(raw, __cte := cleaned)\n" + "final AS (SELECT COUNT(*) AS live_rows FROM cleaned__live)\nSELECT * FROM final\n" +) +_CTE_LIST_TEST: str = ( + 'TEST (mode macro, name "clean_orders_named_twice");\n\nWITH\n' + "input_orders AS (\n" + " SELECT 1 AS order_id, TIMESTAMP '2026-01-01' AS updated_at, FALSE AS is_deleted\n" + " UNION ALL SELECT 1, TIMESTAMP '2026-01-02', FALSE\n),\n" + "@sql.clean_orders(input_orders, __cte := first_pass)\n" + "@sql.clean_orders(first_pass, __cte := second_pass)\n" + "__macro_actual__ AS (SELECT order_id, updated_at FROM second_pass),\n" + "__macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroBuildTestCase( + description="the_same_cte_list_macro_called_twice_with_cte_names", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + query="SELECT order_id, amount_cents FROM orders ORDER BY order_id", + expected_rows=[(1, 1300), (2, 500)], + ), + SqlMacroBuildTestCase( + description="a_caller_reads_a_prefixed_intermediate_cte", + files={**CLEAN_ORDERS, **orders_model(_INTERMEDIATE)}, + query="SELECT live_rows FROM orders", + expected_rows=[(3,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_under_its_own_name_as_alias", + files=picked_orders( + "WITH live AS (SELECT * FROM @src),\nlatest AS (SELECT live.order_id FROM live AS live)" + ), + query="SELECT COUNT(*) FROM orders", + expected_rows=[(4,)], + ), + SqlMacroBuildTestCase( + description="an_alias_named_like_a_cte_elsewhere", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 1),\nlatest AS (SELECT live.order_id FROM live JOIN @src AS other ON live.order_id = other.order_id),\nfinal AS (SELECT o.order_id FROM latest AS o JOIN @src AS live ON live.order_id = o.order_id)" + ), + query="SELECT COUNT(*) FROM orders", + expected_rows=[(8,)], + ), + SqlMacroBuildTestCase( + description="a_nested_with_shadowing_a_cte", + files=picked_orders( + "WITH live AS (SELECT * FROM @src),\nlatest AS (WITH live AS (SELECT 42 AS order_id) SELECT order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(42,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_derived_table", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT s.order_id FROM (SELECT live.order_id FROM live) AS s)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_set_operation", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT order_id FROM live UNION ALL SELECT live.order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,), (2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM @src AS r JOIN live ON live.order_id = r.order_id)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_correlated_subquery", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT r.order_id FROM @src AS r WHERE EXISTS (SELECT 1 FROM live WHERE live.order_id = r.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_parenthesised_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM (live JOIN @src AS s ON live.order_id = s.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_nested_parenthesised_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM ((live JOIN @src AS s ON live.order_id = s.order_id) JOIN @src AS t ON t.order_id = live.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_nested_cte_reading_the_cte_it_shadows", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (WITH live AS (SELECT * FROM live WHERE order_id > 0) SELECT order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_cte_names_when_building_then_each_call_inserts_its_own_ctes( + test_case: SqlMacroBuildTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project(tmp_path=tmp_path, files=test_case.files, command=("build",)) + + assert result.returncode == 0, output(result) + assert ( + query_duckdb(db_path=project_dir / DATABASE, sql=test_case.query) == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroErrorTestCase( + description="cte_on_an_expression_macro", + files={ + **text_macro("m", "VARCHAR", "UPPER(@value)"), + **orders_model('SELECT @sql.m(status, __cte := x) AS v FROM __ref("stg_orders")\n'), + }, + expected_code="P017", + expected_fragments=( + "__cte names the CTEs a CTE-list macro inserts; m() is an expression macro", + ), + ), + SqlMacroErrorTestCase( + description="cte_that_is_not_a_bare_name", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + '@sql.clean_orders(raw, __cte := "gb orders")\nSELECT * FROM raw\n' + ), + }, + expected_code="P017", + expected_fragments=( + '__cte of macro clean_orders() must be a bare CTE name; got "gb orders"', + ), + ), + SqlMacroErrorTestCase( + description="the_same_macro_twice_without_cte_suggests_it", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.clean_orders(raw)\n@sql.clean_orders(raw)\nSELECT * FROM latest\n" + ), + }, + expected_code="P018", + expected_fragments=( + 'CTE "live" from macro clean_orders() clashes with a CTE from macro clean_orders()', + "pass __cte := to give the CTEs one call inserts their own names", + ), + ), + SqlMacroErrorTestCase( + description="a_cte_name_the_caller_already_defines", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "picked AS (SELECT 1 AS x),\n" + "@sql.clean_orders(raw, __cte := picked)\nSELECT * FROM picked\n" + ), + }, + expected_code="P018", + expected_fragments=( + 'CTE "picked" from macro clean_orders() clashes with a CTE already defined', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_invalid_cte_name_when_compiling_then_the_error_names_the_problem( + test_case: SqlMacroErrorTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == 1, output(result) + assert f"error[{test_case.expected_code}]" in output(result), output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCommandTestCase( + description="macro_mode_test_with_cte_names", + files={ + **ALL_MACROS, + **orders_model(_TWO_CALLS), + "tests/unit/test_clean_orders.sql": _CTE_LIST_TEST, + }, + command=("test",), + expected_returncode=0, + expected_fragments=("PASS=1", "clean_orders_named_twice"), + ), + SqlMacroCommandTestCase( + description="lineage_reads_through_renamed_ctes", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("lineage", "orders.amount_cents", "--format", "list"), + expected_returncode=0, + expected_fragments=("stg_orders.amount_cents -> orders.amount_cents",), + ), + SqlMacroCommandTestCase( + description="contracts_type_columns_through_renamed_ctes", + files={ + **CLEAN_ORDERS, + **contract_model( + "orders", "order_id (type BIGINT), amount_cents (type BIGINT)", _TWO_CALLS + ), + }, + command=("compile", "--no-cache"), + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCommandTestCase( + description="format_keeps_the_cte_argument", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("format",), + expected_returncode=0, + expected_fragments=( + "@sql.clean_orders(gb_raw, __cte := gb_orders)", + "@sql.clean_orders(fr_raw, __cte := fr_orders)", + ), + inspected_paths=("models/marts/orders.sql",), + ), + SqlMacroCommandTestCase( + description="lint_sees_renamed_ctes_as_hand_written", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("check",), + expected_returncode=0, + expected_fragments=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_cte_names_when_running_a_command_then_renamed_ctes_act_as_written( + test_case: SqlMacroCommandTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=test_case.command + ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py index e691bff1e8..0e4a440594 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py @@ -339,13 +339,19 @@ ), ), SqlMacroErrorTestCase( - description="expression_macro_without_returns", + description="returns_that_disagrees_with_the_body_is_reported_at_the_macro_file", files={ - "macros/sql/x.sql": _X_HEADER.format(args="v BIGINT", returns="") + "@v + 1\n", - **_CALL_X, + "macros/sql/x.sql": _X_HEADER.format(args="v VARCHAR", returns=", returns BIGINT") + + "UPPER(@v)\n", + **orders_model("SELECT @sql.x('a') AS a\n"), }, - expected_code="P015", - expected_fragments=("expression macro x() needs returns in its MACRO header",), + expected_code="P024", + expected_fragments=( + "macro x() declares returns BIGINT, but its body is VARCHAR\n" + " model: orders\n" + " --> macros/sql/x.sql:3:1", + "declare returns VARCHAR, CAST the body to BIGINT, or remove returns", + ), ), SqlMacroErrorTestCase( description="untyped_argument", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py new file mode 100644 index 0000000000..370e4570ec --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -0,0 +1,616 @@ +"""E2E coverage for optional SQL macro returns types and the check of returns against the body.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, + SqlMacroWarmCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + adapter_project, + contract_model, + orders_model, + output, + run_project, + text_macro, + untyped_text_macro, + with_rules, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_UNTYPED_UPSTREAM: dict[str, str] = { + "models/marts/up.sql": ( + 'MODEL (description "Up");\n\nSELECT NULL AS s FROM __ref("stg_orders") AS o\n' + ), +} +_UNTYPED_CALL: str = 'SELECT @sql.m(u.s) AS v FROM __ref("up") AS u\n' +_FIRST_OF: dict[str, str] = { + "macros/sql/first_of.sql": ( + 'MACRO (\n description "The first of two values",\n' + " arguments (value ANY, other ANY),\n returns TYPE OF value,\n);\n\n" + "list_extract([@value, @other], 1)\n" + ) +} +_FIRST_OF_UNTYPED: dict[str, str] = { + "macros/sql/first_of.sql": _FIRST_OF["macros/sql/first_of.sql"].replace( + " returns TYPE OF value,\n", "" + ) +} +_DAY: str = "CAST(o.{column} AS DATE)" +_DAYS: str = f"{_DAY.format(column='created_at')}, {_DAY.format(column='updated_at')}" +_TYPE_OF_POSITIONS: str = ( + "WITH c AS (\n" + f" SELECT o.order_id, @sql.first_of({_DAYS}) AS in_cte\n" + ' FROM __ref("stg_orders") AS o\n' + ")\n" + "SELECT\n" + " @sql.first_of(o.status, o.country_code) AS text_value,\n" + f" @sql.first_of({_DAYS}) AS day_value,\n" + " c.in_cte,\n" + f" COALESCE(@sql.first_of({_DAYS}), {_DAY.format(column='created_at')}) AS nested\n" + 'FROM __ref("stg_orders") AS o\nJOIN c ON c.order_id = o.order_id\n' +) +_TYPE_OF_COLUMNS: str = ( + "text_value (type VARCHAR), day_value (type DATE), in_cte (type DATE), nested (type DATE)" +) +_TYPE_OF_UNION: str = ( + 'SELECT CAST(NULL AS DATE) AS day_value FROM __ref("stg_orders") AS o\n' + "UNION ALL\n" + f'SELECT @sql.first_of({_DAYS}) AS day_value FROM __ref("stg_orders") AS o\n' +) +_VARIANT_PATHS: str = ( + "WITH raw AS (SELECT PARSE_JSON('{{\"a\": 1}}') AS payload)\n" + "SELECT @sql.payload_value({path}) AS v FROM raw\n" +) +_SNOWFLAKE_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" + ) +} +_ROUNDED: dict[str, str] = { + "macros/sql/rounded.sql": ( + 'MACRO (\n description "Rounded to places",\n' + " arguments (value ANY NUMERIC, places INT CONST DEFAULT 2),\n" + " returns TYPE OF value,\n);\n\nROUND(@value, @places)\n" + ), + "macros/sql/shouted.sql": ( + 'MACRO (\n description "Shouted",\n arguments (value ANY TEXT),\n' + " returns TYPE OF value,\n);\n\nUPPER(@value)\n" + ), + "macros/sql/month_of.sql": ( + 'MACRO (\n description "Month start",\n arguments (value ANY TEMPORAL),\n' + " returns TYPE OF value,\n);\n\nDATE_TRUNC('month', @value)\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_macro_without_returns_is_typed_from_its_body", + files={ + **untyped_text_macro("m", "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_macro_without_returns_keeps_a_known_body_type_that_disagrees", + files={ + **untyped_text_macro("m", "LENGTH(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'v' inferred as BIGINT",), + ), + SqlMacroCompileTestCase( + description="a_macro_with_matching_returns_compiles", + files={ + **text_macro("m", "VARCHAR", "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_wrong_returns_is_reported_at_the_macro_file", + files={ + **text_macro("m", "BIGINT", "UPPER(@value)"), + **contract_model( + "orders", + "v (type BIGINT)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P024]: macro m() declares returns BIGINT, but its body is VARCHAR\n" + " model: orders\n" + " --> macros/sql/m.sql:7:1", + ), + ), + SqlMacroCompileTestCase( + description="an_any_argument_leaves_the_body_unknown_so_returns_is_trusted", + files={ + **with_rules("SQBRCONTRACT105"), + **text_macro("m", "BIGINT", "@value", argument="ANY"), + **_UNTYPED_UPSTREAM, + **contract_model("orders", "v (type BIGINT)", _UNTYPED_CALL), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_body_with_an_untyped_function_is_not_checked_against_returns", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("m", "FLOAT", "COALESCE(TRY_TO_DOUBLE(@value), 0)"), + **orders_model('SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_guessed_date_trunc_type_on_databricks_is_not_checked_against_returns", + files={ + **adapter_project("databricks"), + **text_macro("m", "TIMESTAMP", "DATE_TRUNC('DAY', @value)", argument="TIMESTAMP"), + **orders_model('SELECT @sql.m(o.created_at) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_guessed_regexp_contains_type_on_bigquery_is_not_checked_against_returns", + files={ + **adapter_project("bigquery"), + **text_macro("m", "BOOL", "REGEXP_CONTAINS(@value, 'a')", argument="VARCHAR"), + **orders_model('SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_user_defined_function_on_bigquery_is_not_checked_against_returns", + files={ + **adapter_project("bigquery"), + **text_macro("m", "STRING", "my_udf(@value)", argument="BIGINT"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_partly_unknown_body_of_another_shape_is_reported", + files={ + **text_macro("m", "BIGINT", "[]", argument="BIGINT"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P024]: macro m() declares returns BIGINT, but its body is UNKNOWN[]", + ), + ), + SqlMacroCompileTestCase( + description="a_returns_of_the_body_shape_fills_its_unknown_parts", + files={ + **text_macro("m", "BIGINT[]", "[]", argument="BIGINT"), + **contract_model( + "orders", + 'v (type "BIGINT[]")', + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_invalid_returns_is_reported_on_postgres", + files={ + **adapter_project("postgres"), + **text_macro("m", "FOO BAR", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'FOO BAR' is not valid for adapter " + "'postgres' SQL analysis dialect 'postgres'", + ), + unexpected_fragments=("__polyglot",), + ), + SqlMacroCompileTestCase( + description="an_invalid_returns_is_reported_on_motherduck", + files={ + **adapter_project("motherduck"), + **text_macro("m", "FOO BAR", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'FOO BAR' is not valid for adapter " + "'motherduck' SQL analysis dialect 'duckdb'", + ), + unexpected_fragments=("__polyglot",), + ), + SqlMacroCompileTestCase( + description="a_returns_decimal_wider_than_the_dialect_holds_is_reported", + files={ + **text_macro("m", "DECIMAL(50, 2)", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'DECIMAL(50, 2)' is not valid for adapter " + "'duckdb' SQL analysis dialect 'duckdb': DECIMAL precision 50 exceeds the target " + "range 1..=38", + ), + ), + SqlMacroCompileTestCase( + description="a_returns_decimal_scale_above_its_precision_is_reported", + files={ + **text_macro("m", "DECIMAL(4, 6)", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=("DECIMAL scale 6 is larger than its precision 4",), + ), + SqlMacroCompileTestCase( + description="an_untyped_call_without_returns_suggests_returns_on_k003", + files={ + **untyped_text_macro("m", "@value", argument="ANY"), + **_UNTYPED_UPSTREAM, + **contract_model("orders", "v (type BIGINT)", _UNTYPED_CALL), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared BIGINT", + "declare returns BIGINT in SQL macro m(), or add an explicit CAST", + ), + ), + SqlMacroCompileTestCase( + description="an_untyped_call_without_returns_suggests_returns_on_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **untyped_text_macro("m", "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + 'error[SQBRCONTRACT105]: output "v" is calculated without an outer explicit cast', + "Declare returns VARCHAR in SQL macro m(), or wrap the call in CAST(... AS VARCHAR).", + ), + ), + SqlMacroCompileTestCase( + description="a_cast_inside_a_typed_body_is_not_the_outputs_cast", + files={ + **with_rules("SQBRCONTRACT105"), + **text_macro("m", "VARCHAR", "CAST(LENGTH(@value) AS TEXT)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + ], + ids=lambda case: case.description, +) +def test_given_optional_returns_when_compiling_then_calls_are_typed_and_checked( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + for fragment in test_case.unexpected_fragments: + assert fragment not in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="returns_type_of_gives_each_call_its_argument_type", + files={ + **_FIRST_OF, + **contract_model("orders", _TYPE_OF_COLUMNS, _TYPE_OF_POSITIONS), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_types_a_later_union_branch", + files={ + **with_rules("SQBRCONTRACT105"), + **_FIRST_OF, + **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="without_returns_the_same_calls_stay_untyped", + files={ + **_FIRST_OF_UNTYPED, + **contract_model("orders", _TYPE_OF_COLUMNS, _TYPE_OF_POSITIONS), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'text_value' type could not be proven against declared " + "VARCHAR", + "warning[K003]: column 'day_value' type could not be proven against declared DATE", + "warning[K003]: column 'in_cte' type could not be proven against declared DATE", + "declare returns DATE in SQL macro first_of()", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_cast_variant_path_is_variant", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("payload_value", "TYPE OF value", "@value", argument="ANY"), + **contract_model( + "orders", + "v (type VARIANT)", + _VARIANT_PATHS.format(path="raw.payload:a::VARIANT"), + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_bare_variant_path_stays_unknown_like_the_path", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("payload_value", "TYPE OF value", "@value", argument="ANY"), + **contract_model( + "orders", "v (type VARIANT)", _VARIANT_PATHS.format(path="raw.payload:a") + ), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared VARIANT", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_an_unknown_argument_stays_unknown", + files={ + **_FIRST_OF, + **contract_model( + "orders", + "v (type VARCHAR)", + "SELECT @sql.first_of(bar(o.amount_cents, 0, 10), o.status) AS v\n" + 'FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared VARCHAR", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_keeps_a_known_body_type_that_differs", + files={ + **text_macro("m", "TYPE OF value", "LENGTH(@value)", argument="ANY"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'v' inferred as BIGINT",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_an_unknown_argument_name_is_reported_at_the_macro_file", + files={ + **text_macro("m", "TYPE OF missing", "@value", argument="ANY"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns TYPE OF missing names no argument of macro m()", + "macros/sql/m.sql:4:", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_list_argument_is_reported_at_the_macro_file", + files={ + "macros/sql/m.sql": ( + 'MACRO (\n description "Listed",\n arguments (codes LIST VARCHAR),\n' + " returns TYPE OF codes,\n);\n\n'GB' IN @codes\n" + ), + **contract_model( + "orders", + "v (type BOOLEAN)", + "SELECT @sql.m(['GB']) AS v FROM __ref(\"stg_orders\") AS o\n", + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns TYPE OF codes names a LIST VARCHAR argument of macro m()", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_returns_type_of_when_compiling_then_each_call_has_its_argument_type( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_call_of_each_family_in_several_positions", + files={ + **_ROUNDED, + **contract_model( + "orders", + "amount (type BIGINT), loud (type VARCHAR), month (type TIMESTAMP), " + "in_cte (type DOUBLE)", + "WITH c AS (\n" + " SELECT o.order_id, @sql.rounded(CAST(o.amount_cents AS DOUBLE)) AS in_cte\n" + ' FROM __ref("stg_orders") AS o\n)\n' + "SELECT @sql.rounded(o.amount_cents, 0) AS amount,\n" + " @sql.shouted(o.status) AS loud,\n" + " @sql.month_of(o.created_at) AS month,\n" + " c.in_cte\n" + 'FROM __ref("stg_orders") AS o JOIN c ON c.order_id = o.order_id\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_unknown_argument_type_is_accepted", + files={ + **_ROUNDED, + **orders_model( + "SELECT @sql.rounded(bar(o.amount_cents, 0, 10)) AS v " + 'FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=0, + expected_fragments=("0 errors",), + ), + SqlMacroCompileTestCase( + description="a_wrong_family_argument_is_reported_at_the_call", + files={ + **_ROUNDED, + **orders_model( + 'SELECT @sql.rounded(o.status) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "argument value of macro rounded() expects a number; o.status is VARCHAR", + "models/marts/orders.sql:3:", + ), + ), + SqlMacroCompileTestCase( + description="a_wrong_family_literal_is_reported_at_the_call", + files={ + **_ROUNDED, + **orders_model("SELECT @sql.shouted(12) AS v\n"), + }, + expected_returncode=1, + expected_fragments=( + "error[P017]: argument value of macro shouted() expects text; 12 is an integer", + ), + ), + SqlMacroCompileTestCase( + description="a_body_wrong_for_its_family_is_reported_at_the_macro_file", + files={ + "macros/sql/m.sql": ( + 'MACRO (\n description "Shouted number",\n arguments (value ANY NUMERIC),\n' + ");\n\nUPPER(@value)\n" + ), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P025]: body of macro m() does not accept every value of value ANY NUMERIC", + "--> macros/sql/m.sql:6:1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_any_type_families_when_compiling_then_calls_and_bodies_are_checked( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroWarmCompileTestCase( + description="removing_returns_type_of", + files={ + **_FIRST_OF, + **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), + }, + edited_path="macros/sql/first_of.sql", + replaced_text=(" returns TYPE OF value,\n", ""), + expected_cold_fragment="0 errors, 0 warnings", + expected_warm_fragment="warning[K003]: column 'day_value'", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_changed_returns_form_when_compiling_warm_then_the_cache_is_not_reused( + test_case: SqlMacroWarmCompileTestCase, tmp_path: Path +) -> None: + project_dir, cold = run_project(tmp_path=tmp_path, files=test_case.files, command=("compile",)) + macro: Path = project_dir / test_case.edited_path + old, new = test_case.replaced_text + macro.write_text(macro.read_text(encoding="utf-8").replace(old, new), encoding="utf-8") + warm: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert test_case.expected_cold_fragment in output(cold), output(cold) + assert test_case.expected_warm_fragment in output(warm), output(warm) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py index 9620ef8e76..6d837cd889 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py @@ -202,7 +202,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( unexpected_fragments=_UNPROVEN, ), SqlMacroCompileTestCase( - description="a_call_inside_a_larger_expression_is_typed_from_its_sql", + description="a_call_inside_a_larger_expression_is_typed_but_needs_a_cast_for_the_rule", files={ **_TYPED_RULES_AND_MACROS, **contract_model( @@ -216,14 +216,13 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( }, expected_returncode=1, expected_fragments=( - "warning[K003]: column 'status_bar' type could not be proven against declared " - "VARCHAR", 'error[SQBRCONTRACT105]: output "status_bar" is calculated without an outer ' "explicit cast", ), + unexpected_fragments=("K003",), ), SqlMacroCompileTestCase( - description="a_time_body_keeps_its_inferred_type", + description="a_time_body_disagrees_with_returns_at_the_macro_file", files={ **text_macro("m", "VARCHAR", "CAST('10:00:00' AS TIME)"), **contract_model( @@ -233,10 +232,12 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'v' inferred as",), + expected_fragments=( + "error[P024]: macro m() declares returns VARCHAR, but its body is TIME", + ), ), SqlMacroCompileTestCase( - description="an_array_body_keeps_its_inferred_type", + description="an_array_body_disagrees_with_returns_at_the_macro_file", files={ **text_macro("m", "VARCHAR", "[@value]"), **contract_model( @@ -246,7 +247,9 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'v' inferred as",), + expected_fragments=( + "error[P024]: macro m() declares returns VARCHAR, but its body is", + ), ), SqlMacroCompileTestCase( description="a_longer_text_body_keeps_its_inferred_type", @@ -407,7 +410,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( description="an_untyped_upstream_column_in_a_cte_takes_returns", files={ **with_rules("SQBRCONTRACT105"), - **text_macro("m", "BIGINT", "@value"), + **text_macro("m", "BIGINT", "@value", argument="ANY"), "models/marts/up.sql": ( 'MODEL (description "Up");\n\nSELECT NULL AS s FROM __ref("stg_orders") AS o\n' ), @@ -439,7 +442,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( unexpected_fragments=_UNPROVEN, ), SqlMacroCompileTestCase( - description="an_inferred_type_of_another_family_still_disagrees_through_a_cte", + description="an_inferred_type_of_another_family_disagrees_at_the_macro_file", files={ **text_macro("shouted", "BIGINT", "UPPER(@value)"), **contract_model( @@ -451,7 +454,9 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'shouted' inferred as",), + expected_fragments=( + "error[P024]: macro shouted() declares returns BIGINT, but its body is VARCHAR", + ), ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py new file mode 100644 index 0000000000..32bd9cb50d --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -0,0 +1,554 @@ +"""E2E matrix of SQL macro calls typed by their returns in every position a call can stand.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, + SqlMacroLineageTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + BRACKET, + adapter_project, + contract_model, + inspected_text, + lineage_edges, + orders_model, + output, + run_project, + text_macro, + typed_position_files, + untyped_text_macro, + with_rules, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + +_FROM: str = 'FROM __ref("stg_orders") AS o' +_COMPILED: str = "target/compiled/models/marts/orders.sql" +_CLEAN: tuple[str, ...] = ("0 errors, 0 warnings",) +_MACROS: dict[str, str] = { + **text_macro("status_bar", "VARCHAR", "bar(LENGTH(@value), 0, 10)"), + **text_macro("same", "VARCHAR", "@value", argument="ANY"), + **text_macro("shouted", "VARCHAR", "UPPER(@value)"), +} +_UNKNOWN_CALL: str = "@sql.status_bar(o.status)" +_KNOWN_CALL: str = "@sql.same(o.order_id)" +_DISAGREEING_POSITIONS: tuple[tuple[str, str, str, str], ...] = ( + ("main_whole_item", "contracted", f"SELECT {{m}} AS x {_FROM}\n", "VARCHAR"), + ( + "cte_item_read_by_name", + "contracted", + f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT x FROM c\n", + "VARCHAR", + ), + ( + "star_from_a_cte", + "contracted", + f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT * FROM c\n", + "VARCHAR", + ), + ( + "derived_table", + "contracted", + f"SELECT s.x FROM (SELECT {{m}} AS x {_FROM}) AS s\n", + "VARCHAR", + ), + ( + "nested_with", + "contracted", + f"WITH c AS (WITH d AS (SELECT {{m}} AS x {_FROM}) SELECT x FROM d)\nSELECT x FROM c\n", + "VARCHAR", + ), + ( + "union_both_branches", + "contracted", + f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + "VARCHAR", + ), + ( + "after_a_star_modifier", + "downstream", + "WITH c AS (SELECT o.order_id, o.status " + _FROM + ")\n" + "SELECT * EXCLUDE (order_id), {m} AS x FROM c AS o\n", + "VARCHAR", + ), +) +_AGREEING_POSITIONS: tuple[tuple[str, str, str, str], ...] = ( + ( + "scalar_subquery", + "contracted", + f"SELECT (SELECT {{m}} {_FROM} LIMIT 1) AS x\n", + "VARCHAR", + ), + ( + "union_right_branch", + "contracted", + f"SELECT o.status AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + "VARCHAR", + ), + ( + "union_left_branch", + "contracted", + f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT o.status AS x {_FROM}\n", + "VARCHAR", + ), + ( + "parenthesised_branch", + "contracted", + f"SELECT CAST(NULL AS VARCHAR) AS x {_FROM}\nUNION ALL\n(SELECT {{m}} AS x {_FROM})\n", + "VARCHAR", + ), + ("arithmetic", "contracted", f"SELECT LENGTH({{m}}) + 1 AS x {_FROM}\n", "BIGINT"), + ("coalesce", "contracted", f"SELECT COALESCE({{m}}, {{m}}) AS x {_FROM}\n", "VARCHAR"), + ( + "case", + "contracted", + f"SELECT CASE WHEN o.order_id > 1 THEN {{m}} ELSE NULL END AS x {_FROM}\n", + "VARCHAR", + ), + ("function_argument", "contracted", f"SELECT LENGTH({{m}}) AS x {_FROM}\n", "BIGINT"), + ("aggregate_over_the_call", "contracted", f"SELECT MAX({{m}}) AS x {_FROM}\n", "VARCHAR"), + ( + "window", + "contracted", + f"SELECT MAX({{m}}) OVER (ORDER BY o.order_id) AS x {_FROM}\n", + "VARCHAR", + ), + ("unaliased_call", "uncontracted", f"SELECT {{m}} {_FROM}\n", "VARCHAR"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description=description, + files={ + **_MACROS, + **typed_position_files(shape, sql.replace("{m}", _UNKNOWN_CALL), declared), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ) + for description, shape, sql, declared in _DISAGREEING_POSITIONS + _AGREEING_POSITIONS + ], + ids=lambda case: case.description, +) +def test_given_an_unknown_body_when_compiling_then_the_call_takes_its_returns_type( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description=description, + files={ + **_MACROS, + **typed_position_files(shape, sql.replace("{m}", _KNOWN_CALL), declared), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'x' inferred as",), + ) + for description, shape, sql, declared in _DISAGREEING_POSITIONS + ], + ids=lambda case: case.description, +) +def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stands( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="aggregate_inside_the_body", + files={ + **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), + **contract_model( + "orders", + "status (type VARCHAR), x (type BIGINT)", + f"SELECT o.status, @sql.summed(o.status) AS x {_FROM}\nGROUP BY o.status\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="aggregate_inside_the_body_still_needs_group_by", + files={ + **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), + **orders_model(f"SELECT o.status, @sql.summed(o.status) AS x {_FROM}\n"), + }, + expected_returncode=1, + expected_fragments=("error[B230]",), + ), + SqlMacroCompileTestCase( + description="a_nested_call_types_the_macro_calling_it", + files={ + **_MACROS, + **untyped_text_macro("outer_bar", "@sql.status_bar(@value)"), + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.outer_bar(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_any_argument_of_unknown_type_takes_returns", + files={ + **_MACROS, + **contract_model( + "orders", + "x (type VARCHAR)", + f"SELECT @sql.same(bar(o.order_id, 0, 10)) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_known_identical_body", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.shouted(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_call_inside_a_python_macro_argument_is_inferred_only", + files={ + **_MACROS, + **BRACKET, + **contract_model( + "orders", + "x (type VARCHAR)", + f"SELECT @py.bracket(@sql.status_bar(o.status)) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_outer_cast_wins_and_satisfies_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **_MACROS, + **contract_model( + "orders", + "x (type BIGINT)", + f"SELECT CAST(@sql.status_bar(o.status) AS BIGINT) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_call_inside_arithmetic_needs_a_cast_for_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **_MACROS, + **contract_model( + "orders", + "x (type BIGINT)", + f"SELECT LENGTH(@sql.status_bar(o.status)) + 1 AS x {_FROM}\n", + ), + }, + expected_returncode=1, + expected_fragments=( + 'error[SQBRCONTRACT105]: output "x" is calculated without an outer explicit cast', + "Wrap the complete expression in CAST(... AS BIGINT)", + ), + ), + SqlMacroCompileTestCase( + description="a_binding_error_after_a_call_is_located_where_it_was_written", + files={ + **_MACROS, + **orders_model(f"SELECT @sql.status_bar(o.status) AS x, o.missing {_FROM}\n"), + }, + expected_returncode=1, + expected_fragments=("--> models/marts/orders.sql:3:42",), + ), + SqlMacroCompileTestCase( + description="the_reserved_analysis_function_is_rejected_in_a_model", + files={**orders_model(f"SELECT __polyglot_typed(o.status, 'VARCHAR') AS x {_FROM}\n")}, + expected_returncode=1, + expected_fragments=("uses __polyglot_typed, a name reserved for SQL analysis",), + ), + SqlMacroCompileTestCase( + description="a_downstream_model_sees_the_returns_type", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" + ), + **contract_model( + "order_labels", "x (type BIGINT)", 'SELECT b.x FROM __ref("orders") AS b\n' + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'x' inferred as",), + ), + SqlMacroCompileTestCase( + description="a_grouping_key_written_through_a_call", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT o.amount_cents + 1 AS k, COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_in_select_and_grouping_key", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT @sql.plus_one(o.amount_cents) AS k, COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_selected_and_grouped_written_out", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT @sql.plus_one(o.amount_cents) AS k, COUNT(*) AS n {_FROM} GROUP BY o.amount_cents + 1\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_in_a_rollup", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT o.amount_cents + 1 AS k, COUNT(*) AS n {_FROM} GROUP BY ROLLUP (@sql.plus_one(o.amount_cents))\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_having_over_a_called_grouping_key", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents) HAVING o.amount_cents + 1 > 0\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_distinct_ordered_by_a_call", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT DISTINCT o.amount_cents + 1 AS k {_FROM} ORDER BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_in_a_keyword_slot_keeps_the_analysis_whole", + files={ + **text_macro("part", "VARCHAR", "YEAR"), + **contract_model( + "orders", + "y (type BIGINT)", + f"SELECT EXTRACT(@sql.part(o.status) FROM o.created_at) AS y {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("Semantic checks were partial",), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_snowflake_dateadd_part", + files={ + **adapter_project("snowflake"), + **text_macro("p", "VARCHAR", "day"), + **orders_model(f"SELECT DATEADD(@sql.p(o.status), 1, o.created_at) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_snowflake_datediff_part", + files={ + **adapter_project("snowflake"), + **text_macro("p", "VARCHAR", "day"), + **orders_model( + f"SELECT DATEDIFF(@sql.p(o.status), o.created_at, o.updated_at) AS y {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_an_interval_unit", + files={ + **text_macro("p", "VARCHAR", "DAY"), + **orders_model(f"SELECT o.created_at + INTERVAL 1 @sql.p(o.status) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_bare_cast_type", + files={ + **text_macro("t", "VARCHAR", "BIGINT"), + **orders_model(f"SELECT CAST(o.order_id AS @sql.t(o.status)) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_parameterised_cast_type", + files={ + **text_macro("t", "VARCHAR", "DECIMAL(18, 2)"), + **orders_model(f"SELECT CAST(o.order_id AS @sql.t(o.status)) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="the_compiled_sql_has_no_marker", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("bar(LENGTH(o.status), 0, 10) AS x",), + unexpected_fragments=("__polyglot",), + inspected_paths=(_COMPILED,), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_typed_call_case_when_compiling_then_it_reports_as_expected( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed + + +@pytest.mark.xfail( + reason="a * REPLACE item takes its source column's type; a known polyglot gap", strict=True +) +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_call_in_a_star_replace", + files={ + **_MACROS, + **contract_model( + "orders", + "order_id (type VARCHAR)", + "WITH c AS (SELECT o.order_id " + _FROM + ")\n" + "SELECT * REPLACE (@sql.status_bar(CAST(order_id AS VARCHAR)) AS order_id) " + "FROM c\n", + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_call_in_a_star_replace_when_compiling_then_it_takes_its_returns_type( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroLineageTestCase( + description="an_identity_macro", + macro_files={ + **text_macro("ident", "BIGINT", "@value", argument="BIGINT"), + **orders_model(f"SELECT @sql.ident(o.order_id) AS x {_FROM}\n"), + }, + inline_files=orders_model(f"SELECT o.order_id AS x {_FROM}\n"), + column="orders.x", + expected_edge="stg_orders.order_id -> orders.x", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_call_when_reading_lineage_then_it_matches_the_inline_sql( + test_case: SqlMacroLineageTestCase, tmp_path: Path +) -> None: + macro_dir, _ = run_project( + tmp_path=tmp_path / "macro", files=test_case.macro_files, command=("compile", "--no-cache") + ) + inline_dir, _ = run_project( + tmp_path=tmp_path / "inline", + files=test_case.inline_files, + command=("compile", "--no-cache"), + ) + lineage: tuple[str, ...] = ("--no-color", "lineage", test_case.column, "--format", "list") + through_macro: subprocess.CompletedProcess[str] = run_sqb( + command=lineage, project_dir=macro_dir + ) + inline: subprocess.CompletedProcess[str] = run_sqb(command=lineage, project_dir=inline_dir) + + assert through_macro.returncode == 0, output(through_macro) + assert test_case.expected_edge in through_macro.stdout, output(through_macro) + assert lineage_edges(through_macro.stdout) == lineage_edges(inline.stdout), output(inline) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 2e0e0b5f98..a10d18b03e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -788,6 +788,42 @@ def test_given_unwritable_cache_when_testing_since_then_warns_and_succeeds( unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), git_after_edits=("commit", "--quiet", "--all", "-m", "edit"), ), + SinceSelectionE2ETestCase( + description="changing a sql macro's returns form selects the test of its caller", + edits={ + _OUTER_SQL_MACRO: _SQL_MACROS[_OUTER_SQL_MACRO].replace( + "returns DOUBLE", "returns TYPE OF cents" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed", + "test_order_net\n executes macro net_dollars (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="naming the CTEs of a CTE-list sql macro call selects the caller's test", + edits={ + "models/_macros/sql/live_orders.sql": ( + "MACRO (\n description 'Orders still open.',\n" + " arguments (src RELATION (order_id INTEGER, status VARCHAR)),\n);\n\n" + "WITH live AS (SELECT * FROM @src WHERE status = 'open')\n" + ), + "models/order_live.sql": ( + "MODEL (description 'Live orders.');\n\n" + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.live_orders(raw, __cte := open_orders)\n" + "SELECT order_id FROM open_orders\n" + ), + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=("Changed since main: 1 model, 1 macro added",), + unexpected_fragments=("FAIL=1",), + ), SinceSelectionE2ETestCase( description="reformatting, commenting and redescribing a sql macro changes nothing", edits={ diff --git a/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json b/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json new file mode 100644 index 0000000000..716f276699 --- /dev/null +++ b/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P025", + "message": "body of macro x() does not accept every value of v ANY NUMERIC: Function 'upper' argument 1 expects String, found numeric\n model: stg_orders\n --> models/staging/_sqlbuild/_macros/sql/x.sql:3:1\n --> models/staging/stg_orders.sql:5:31", + "help": "an ANY family argument may be any type of its family; the body is checked with the family's widest type, so CAST the argument or narrow its declared type" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json b/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json index 569a956a00..360c1a6978 100644 --- a/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json +++ b/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json @@ -9,7 +9,7 @@ "severity": "error", "code": "P018", "message": "CTE \"live\" from macro live_orders() clashes with a CTE already defined in model:stg_orders\n model: stg_orders\n --> models/staging/stg_orders.sql:7:1\n --> models/staging/_sqlbuild/_macros/sql/live_orders.sql:6:6", - "help": "rename one of the CTEs; a model can't call a CTE-list macro whose CTE names it already uses" + "help": "pass __cte := to give the CTEs one call inserts their own names, as in @sql.live_orders(..., __cte := first_orders), or rename one of the CTEs" } ] } diff --git a/tests/goldens/compiler/failure/sql-macro-invalid-definition.json b/tests/goldens/compiler/failure/sql-macro-invalid-definition.json index bce56b86fc..a85b291acf 100644 --- a/tests/goldens/compiler/failure/sql-macro-invalid-definition.json +++ b/tests/goldens/compiler/failure/sql-macro-invalid-definition.json @@ -8,8 +8,8 @@ "phase": "compile", "severity": "error", "code": "P015", - "message": "expression macro x() needs returns in its MACRO header\n --> models/staging/_sqlbuild/_macros/sql/x.sql:1:1", - "help": "declare the type the expression returns, such as returns DECIMAL(18, 2)" + "message": "returns TYPE OF w names no argument of macro x()\n --> models/staging/_sqlbuild/_macros/sql/x.sql:1:63", + "help": "name one of its arguments: v" } ] } diff --git a/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json b/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json new file mode 100644 index 0000000000..e9042051be --- /dev/null +++ b/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P024", + "message": "macro x() declares returns DOUBLE, but its body is VARCHAR\n model: stg_orders\n --> models/staging/_sqlbuild/_macros/sql/x.sql:3:1\n --> models/staging/stg_orders.sql:5:31", + "help": "declare returns VARCHAR, CAST the body to DOUBLE, or remove returns to use the inferred type" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json index a7ccced078..30eded6455 100644 --- a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json +++ b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json @@ -12497,16 +12497,16 @@ [ "type", { -"DuckDB": "9c4e957c3856f1159283313a", +"DuckDB": "8731ee5ed03816bf6b4a8464", "None": "8731ee5ed03816bf6b4a8464", "bigquery": "657a69dabb6a9b2c7b834d41", "databricks": "606d813c3f1dfcf4921045c8", -"duckdb": "9c4e957c3856f1159283313a", +"duckdb": "8731ee5ed03816bf6b4a8464", "generic": "8731ee5ed03816bf6b4a8464", "motherduck": "901c5133ed47d4769286e860", "mysql": "b9f1a78bcda3bc968346cc10", -"postgres": "9c4e957c3856f1159283313a", -"postgresql": "9c4e957c3856f1159283313a", +"postgres": "8731ee5ed03816bf6b4a8464", +"postgresql": "8731ee5ed03816bf6b4a8464", "snowflake": "cd2eb5ae8b54388fde5d5737", "tsql": "8619573249fac6f54d0bc771" } @@ -12718,16 +12718,16 @@ [ "type", { -"DuckDB": "0da0fad3107f7a26744c9411", +"DuckDB": "4172129e8d84a0da597bc612", "None": "4172129e8d84a0da597bc612", "bigquery": "d138b3b3355094a33a14355a", "databricks": "4172129e8d84a0da597bc612", -"duckdb": "0da0fad3107f7a26744c9411", +"duckdb": "4172129e8d84a0da597bc612", "generic": "4172129e8d84a0da597bc612", "motherduck": "901c5133ed47d4769286e860", "mysql": "6c03a3450ea4742ee7396e3c", -"postgres": "0da0fad3107f7a26744c9411", -"postgresql": "0da0fad3107f7a26744c9411", +"postgres": "4172129e8d84a0da597bc612", +"postgresql": "4172129e8d84a0da597bc612", "snowflake": "e21aafb74f6e5c5e181abd5e", "tsql": "2e261f2f5a01e0e0f4a34a7a" } @@ -16458,16 +16458,16 @@ [ "type", { -"DuckDB": "e1554f02dea662c971e7b9b9", +"DuckDB": "6f851103102e7b986535376f", "None": "6f851103102e7b986535376f", "bigquery": "58b7e0908df666938398205f", "databricks": "6f851103102e7b986535376f", -"duckdb": "e1554f02dea662c971e7b9b9", +"duckdb": "6f851103102e7b986535376f", "generic": "6f851103102e7b986535376f", "motherduck": "901c5133ed47d4769286e860", "mysql": "1a853242fb56557e2865a86a", -"postgres": "a0b34c5b4e5ace89b22c6971", -"postgresql": "a0b34c5b4e5ace89b22c6971", +"postgres": "6f851103102e7b986535376f", +"postgresql": "6f851103102e7b986535376f", "snowflake": "87800133cac38d9119180c0a", "tsql": "0e9ef5373e5d52970e8e1656" } diff --git a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py index ee1cdf607d..79cb00a903 100644 --- a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py +++ b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py @@ -100,6 +100,15 @@ def test_given_generated_types_when_normalizing_then_every_known_dialect_answers ), expected_parse_error_logged=False, ), + TypeNormalizationTestCase( + description="a decimal wider than the dialect holds keeps its precision", + type_sql="DECIMAL(39, 2)", + dialect="duckdb", + expected_type=NormalizedType( + normalized_name="DECIMAL(39,2)", family=TypeFamily.DECIMAL, precision=39, scale=2 + ), + expected_parse_error_logged=False, + ), TypeNormalizationTestCase( description="spaced decimal parameters", type_sql=" decimal( 10 , 2 ) ", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 78b5d15fe4..8339b61cba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -777,7 +777,7 @@ def test_given_sql_with_at_tokens_when_finding_macro_calls_then_returns_real_uni 'MACRO (description "Valid", arguments (v BIGINT), returns BIGINT);\n\n@v + 1\n' ), "macros/sql/z_invalid.sql": ( - 'MACRO (description "Invalid", arguments (v BIGINT));\n\n@v + 1\n' + 'MACRO (description "Invalid", returns BIGINT);\n\nWITH a AS (SELECT 1)\n' ), }, expected_exports=("a_valid",), diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 58da2a1492..c87be57cd4 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -52,10 +52,11 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` -`returns` is required for expression macros and not allowed for the other forms. +`returns` is optional for expression macros and not allowed for the other forms. See +[Result types](#result-types) for what it does. The description follows the same rules as every header string: write text that contains double quotes as `"""..."""`, and a wrapped description is stored folded into one flowing text (see @@ -81,7 +82,7 @@ compiles to: ```sql SELECT o.order_id, - ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees + CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM analytics.stg_orders AS o ``` @@ -119,6 +120,28 @@ The compiled model holds `orders_raw`, `live`, `ranked`, `latest` and `final`, f order, and later CTEs may read any of the macro's CTEs. Commas around the call are optional; `sqb format` normalises them. +The last CTE of a CTE-list macro is its result. To call one macro more than once in a model, name +each call's CTEs with the reserved `__cte` argument: + +```sql +WITH eu_raw AS (SELECT * FROM __ref("stg_orders_eu")), +us_raw AS (SELECT * FROM __ref("stg_orders_us")), +@sql.clean_orders(eu_raw, __cte := eu_orders) +@sql.clean_orders(us_raw, __cte := us_orders) +final AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders) +SELECT * FROM final +``` + +The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so +this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, +`us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are +renamed with them, wherever a name reads the macro's CTE: a table alias such as `FROM live AS l`, +or a nested `WITH` that defines its own `live`, keeps its own meaning. Arguments are never renamed, +even one named like a macro CTE. Later CTEs may read the prefixed ones too. + +`__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in +`arguments`. A call without it keeps the macro's own names. + ### Column lists A column-list macro inserts its columns into the caller's select list. Every column needs a name: @@ -140,7 +163,8 @@ Use a column list when the same ordered set of columns repeats, for example in e `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. A CTE or column name a macro inserts that the caller already defines fails with `P018`, which names -both. Nothing is renamed, so a model cannot call the same CTE-list macro twice. +both. Calling the same CTE-list macro twice without `__cte` fails the same way, and the error +suggests `__cte`. ## Parameter types @@ -150,6 +174,7 @@ Every parameter has a type, and may have a `DEFAULT`: |---|---| | `BIGINT`, `VARCHAR`, `TIMESTAMP`, ... | Any SQL expression of a compatible type, evaluated per row: a column, arithmetic, a function call, a literal or a constant. | | `ANY` | Any scalar SQL expression, whatever its type: a column, a literal, an expression or a scalar constant. Its type is not checked. Lists and relations fail with `P017`. | +| `ANY NUMERIC`, `ANY TEXT`, `ANY TEMPORAL` | Like `ANY`, but only values of one family: integer, decimal and float types; string types; or date, time and timestamp types. | | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | @@ -167,6 +192,19 @@ still checked, and the argument's type counts as unknown there. `ANY` is not `VA value of any type. `ANY` takes a `DEFAULT` but not `CONST`, which needs a type to check values against, and it cannot be a `LIST` or `SET` element type. +### Choosing an argument type + +- A fixed type, such as `BIGINT`, when the body needs that type. Callers may pass any compatible + value, and the body is checked with that type. +- A family, such as `ANY NUMERIC`, when the body works for every type of the family, such as + `ROUND(@value, @places)`. A call whose argument has a known type outside the family fails with + `P017` at the argument. The body is checked as if the argument had the family's widest type + (`DOUBLE`, `VARCHAR` or `TIMESTAMP`), so `UPPER(@value)` on `ANY NUMERIC` fails with `P025` at the + macro file. Pair a family with `returns TYPE OF value` to return the caller's own type. +- `ANY` when the body works for values of any type, typically because it casts the argument itself. + +Families take a `DEFAULT` but not `CONST`, and are not `LIST` or `SET` element types. + Leave `CONST` off unless the body needs a literal. `CONST` is part of the macro's contract: callers can no longer pass a column. @@ -195,7 +233,7 @@ These fail with `P022`: - a relation parameter written bare where it would read a table, such as `FROM src` or `src.created_at`; the error says `src is a parameter of macro clean_orders(); write @src` -Parameters may not be named `sql`, `py`, `const`, `enum` or `param`. +Parameters may not be named `sql`, `py`, `const`, `enum`, `param` or `__cte`. ### Positions that need a literal @@ -244,7 +282,7 @@ isn't visible from there fails with `S026`; to use a caller's value, make it an `@const(...)` at the call site. SQL macros a body calls resolve the same way, so a macro can call a folder-local macro beside it that its callers cannot see. -## Checks and tools +## Checks Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: @@ -270,18 +308,48 @@ parameters of a calling macro, and calls in models with runtime placeholders; th column checks still run on the expanded SQL. All other errors also point at the call and at the macro file. -A select-list item that is one expression macro call under an alias, as in -`@sql.cents_to_dollars(o.amount_cents) AS amount`, has the macro's `returns` type wherever the -compiler infers no type for the expanded SQL, or exactly that type. This holds in the main query -and in any top-level CTE, including each branch of their `UNION`, `INTERSECT` or `EXCEPT`, and the -type flows on as an inferred type would: through `SELECT * FROM final`, pass-through columns and -joins, to contract checks (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A -set operation's column combines its branches as usual, so a sibling branch of another type still -leaves it unproven, and a CTE holding one is read by column name, since `SELECT *` over it is not -typed. Such a call needs no `CAST` to satisfy a matching contract column. Any other -inferred type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and `K002` -reports the mismatch. A call inside a larger expression, a subquery, a nested `WITH` or a set -operation with a parenthesised branch is typed from the expanded SQL as before. +## Result types + +An expression macro call is typed wherever it stands, from its expanded SQL, as if the SQL were +written there. `returns` declares the type for when that SQL's type is unknown: + +```sql +MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); + +bar(LENGTH(@value), 0, 10) +``` + +- The declared type applies only where the expanded SQL's type is unknown. A partly unknown type, + such as `[]`, an array of unknown elements, takes `returns` only when `returns` has its shape, + such as `BIGINT[]`. A known type stands, even a close one such as `VARCHAR(100)` for + `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. +- The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each + branch of a set operation, through operators and functions around the call, to contract checks + (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A select-list item that is + one call to a macro with `returns` needs no `CAST` for `SQBRCONTRACT105`; a call inside a larger + expression still does. +- Without `returns`, the call is typed by inference alone. When an output is one call whose type is + unknown, `K003` and `SQBRCONTRACT105` suggest declaring `returns` on the macro. +- `returns TYPE OF ` declares the type of that argument at each call, under the same + rule: `arguments (value ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF value`. If the + argument's own type is unknown, so is the call's. The argument must be a scalar argument of the + macro (`P015`). +- A fixed `returns` is checked against the body at the macro file. The body is typed from the + declared argument types; a known type of another family, such as `UPPER(@value)` with + `returns BIGINT`, fails with `P024`, as does a partly unknown body of another shape, such as `[]` + with `returns BIGINT`. A body whose type cannot be inferred, for example because it reads `ANY` + arguments or calls a function whose type is unknown or only guessed, trusts `returns`. +- `returns` must be a type the adapter's SQL dialect accepts, including a decimal precision and + scale it can hold, such as at most `DECIMAL(38, s)` on DuckDB (`P015`). +- A call where only a keyword can stand, such as the date part of `EXTRACT`, is typed by inference + alone. +- A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed + by inference alone. +- `returns` never changes the compiled SQL. + +A `* REPLACE` item takes its source column's type rather than the call's. + +## Tools - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 604fe7f6d1..b51229cdf8 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -590,7 +590,9 @@ __macro_actual__ AS (SELECT order_id, updated_at FROM latest), __macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at) ``` -Other test modes may not call a CTE-list macro in their `WITH` list. +These calls may pass `__cte := name`, as in a model, to call the macro more than once or to give +its CTEs names of the test's own. Other test modes may not call a CTE-list macro in their `WITH` +list. ### UDF tests