diff --git a/Cargo.lock b/Cargo.lock index f6c7f47891..fa9a7c522f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1191,6 +1191,7 @@ checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" name = "sqlbuild-analysis" version = "0.170.0" dependencies = [ + "memchr", "polyglot-sql", "polyglot-sql-function-catalogs", "rayon", diff --git a/README.md b/README.md index 776d5014b7..ed0a07cd5f 100644 --- a/README.md +++ b/README.md @@ -167,15 +167,16 @@ SELECT o.order_id, o.customer_id, p.amount_cents AS total_cents, - @cents_to_dollars("p.amount_cents") AS total_dollars + @py.cents_to_dollars("p.amount_cents") AS total_dollars FROM __ref("stg_orders") o JOIN __ref("stg_payments") p USING (order_id) ``` -Macros are plain Python functions that return SQL, kept next to the models that use them: +Python macros are plain functions that return SQL, kept next to the models that use them; SQL +macros in `macros/sql/` cover logic-free fragments with typed, checked arguments: ```python -# models/marts/_sqlbuild/_macros/currency.py +# models/marts/_sqlbuild/_macros/python/currency.py def cents_to_dollars(column: str) -> str: """Convert a cents integer column to a dollars decimal with two decimal places.""" return f"ROUND(({column}) / 100.0, 2)" diff --git a/benchmarks/dbt-compare/write_dbt_dense.py b/benchmarks/dbt-compare/write_dbt_dense.py index 6fa7cbb0d7..9c5d33ba0e 100644 --- a/benchmarks/dbt-compare/write_dbt_dense.py +++ b/benchmarks/dbt-compare/write_dbt_dense.py @@ -43,7 +43,7 @@ COLUMN_RE = re.compile( r'^\s+(\w+) \(type (\w+), nullable (true|false), description "([^"]*)"(, audits \[not_null\])?\)' ) -MACRO_CALL_RE = re.compile(r'@(macro_\d{3}_\d{5})\("([^"]*)"\)') +MACRO_CALL_RE = re.compile(r'@py\.(macro_\d{3}_\d{5})\("([^"]*)"\)') CAST_RE = re.compile(r"CAST\(([^ ]+) AS (\w+)\) AS (\w+)") @@ -201,7 +201,7 @@ def write_dbt(sqb: Path, out: Path, n: int) -> Path: base_re = re.compile( r'def (base_offset_\d{3})\(expression: str\) -> str:\n return f"\(\{expression\} \+ 1\)"' ) - for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/*/*.py")): + for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/python/*/*.py")): text = macro.read_text() name, ret = macro_def_re.search(text).groups() offset = offset_re.search(text)[1] diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 034ba496ec..84f9d6cbb5 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -13,6 +13,7 @@ default = [] test-support = [] [dependencies] +memchr.workspace = true polyglot-sql.workspace = true polyglot-sql-function-catalogs.workspace = true rayon.workspace = true diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index ce7fe35a60..757a894a90 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -551,6 +551,7 @@ pub(crate) fn decode_outcome( let binding_validated: bool = decoder.flag()?; decoder.finish()?; let outcome = ModelOutcome { + sql_macro_findings: Vec::new(), analysis: ModelAnalysis { analysis_succeeded, columns, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index ca86762b22..35bd85b0f4 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -30,6 +30,7 @@ use crate::assembly::analysis_session::_helpers::mappings::{ }; use crate::assembly::analysis_session::_helpers::mappings::{producers, waves}; use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE}; +use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use crate::assembly::analysis_session::models::{ AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts, LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables, @@ -38,6 +39,9 @@ use crate::assembly::analysis_session::models::{ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; +use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments; +use crate::sql_macros::models::SqlMacroFinding; +use crate::sql_macros::types::ArgumentCheckInput; /// What an internal native failure of the session names. const SESSION_CONTEXT: &str = "native model analysis"; @@ -279,6 +283,7 @@ impl AnalysisSession { } }; self.outcomes[*model] = Some(ModelOutcome { + sql_macro_findings: Vec::new(), analysis, cleaned_sql: result.cleaned_sql, deferred_cleaning: None, @@ -286,6 +291,48 @@ impl AnalysisSession { fused_binding_validated: true, }); } + self.check_sql_macros(models) + } + + /// Each model's SQL macro call arguments checked against its analysed types, cache hits too. + fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> { + let checked: Vec = models + .iter() + .copied() + .filter(|model| { + let request: &ModelRequest = &self.request.models[*model]; + !request.sql_macro_checks.is_empty() && request.placeholders.is_empty() + }) + .collect(); + self.ensure_cleaned(&checked)?; + let mut inputs: Vec<(usize, String, Shapes)> = Vec::with_capacity(checked.len()); + for model in checked { + let Some(outcome) = self.outcomes[model].as_ref() else { + continue; + }; + let Ok(cleaned) = resolved_cleaned_sql(outcome) else { + continue; + }; + let schema: Shapes = self.binding_schema(&self.request.models[model]); + inputs.push((model, cleaned.into_owned(), schema)); + } + let batch: Vec> = inputs + .iter() + .map(|(model, cleaned, schema)| { + ( + cleaned.as_str(), + schema.as_slice(), + self.request.models[*model].sql_macro_checks.as_slice(), + ) + }) + .collect(); + let findings: Vec> = + check_sql_macro_arguments(&self.request.dialect, &batch); + for ((model, _, _), found) in inputs.iter().zip(findings) { + if let Some(outcome) = self.outcomes[*model].as_mut() { + outcome.sql_macro_findings = found; + } + } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 088c9ee3c9..594038d064 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -12,6 +12,7 @@ use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, Pairs, Shapes}; use crate::semantic_validation::models::DiagnosticRow; +use crate::sql_macros::models::{SqlMacroCheck, SqlMacroFinding}; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. #[derive(Debug, Clone, PartialEq, Eq)] @@ -42,6 +43,8 @@ pub struct ModelRequest { /// The SQL Python's dynamic pivot proof parses: placeholders replaced by their defaults. pub pivot_sql: String, pub dynamic_families: Vec, + /// What the model's analysed SQL must show for its SQL macro calls' arguments to fit. + pub sql_macro_checks: Vec, } /// The reference-derived fields of a `ModelRequest`. @@ -258,6 +261,8 @@ pub struct ModelOutcome { pub deferred_cleaning: Option, pub validated_schema: Shapes, pub fused_binding_validated: bool, + /// The SQL macro call arguments the analysed SQL shows do not fit; never cached. + pub sql_macro_findings: Vec, } /// The finished run of the session: what Python records from it. diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 321905eb3e..459254d0f7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -108,6 +108,7 @@ pub(crate) fn model(name: &str, sql: &str, sources: &[&str], refs: &[&str]) -> M snapshot_columns: None, pivot_sql: sql.to_owned(), dynamic_families: Vec::new(), + sql_macro_checks: Vec::new(), } } diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 71738049d7..7c717f791d 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -67,6 +67,8 @@ pub fn locate_binding_diagnostics( line, column, location, + help: None, + related: None, }); } Ok(located) diff --git a/crates/sqlbuild-analysis/src/binding_locations/models.rs b/crates/sqlbuild-analysis/src/binding_locations/models.rs index efd854bf90..0a843c3302 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/models.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/models.rs @@ -42,4 +42,15 @@ pub struct LocatedDiagnostic { pub line: Option, pub column: Option, pub location: Option, + pub help: Option, + pub related: Option, +} + +/// A second authored location that adds context to a diagnostic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RelatedSpot { + pub label: String, + pub path: String, + pub line: usize, + pub column: usize, } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs index cfa25123d3..16f0a1fe9b 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs @@ -47,6 +47,16 @@ fn call_open(sql: &str, at: usize) -> Option { return None; } let mut cursor = at + 1 + first.len_utf8(); + for prefix in ["py.", "sql."] { + if sql[at + 1..].starts_with(prefix) { + cursor = at + 1 + prefix.len(); + let first = char_at(sql, cursor)?; + if !(first == '_' || first.is_alphabetic()) { + return None; + } + cursor += first.len_utf8(); + } + } while let Some(character) = char_at(sql, cursor) { if !is_identifier_continue(character) { break; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index f98885a529..243271e3df 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -1226,6 +1226,15 @@ fn macro_calls(sql: &str, syntax: &LexicalSyntax) -> Result, String> if byte_at(sql, index) == Some(b'@') && let Some((name, end)) = read_identifier(sql, index + 1, syntax) { + let (name, end) = + if matches!(name.as_str(), "py" | "sql") && byte_at(sql, end) == Some(b'.') { + match read_identifier(sql, end + 1, syntax) { + Some(called) => called, + None => (name, end), + } + } else { + (name, end) + }; let open = space_end(sql, end); if byte_at(sql, open) == Some(b'(') && !DECLARATION_CALLS.contains(&name.as_str()) { if !calls.contains(&name) { diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index 305d2fa2e7..ef30aaadb8 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -56,7 +56,7 @@ pub(crate) fn empty_model_fixture_marker_preserves_direct_mode_validation() -> b pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool { let cases = [ ( - r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#, + r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @py.status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#, "SQL test 'tests/status.sql' must alias every non-trivial __macro_expected__ projection", ), ( diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs index b1125b3969..3936828345 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs @@ -42,7 +42,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' CTE name 'my$helper' must be an unquoted identifier of ASCII letters, digits and underscores", NAME_HELP, @@ -284,7 +284,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call udf; call reusable logic only in __macro_actual__", None, @@ -316,7 +316,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "macro", raw: true, syntax: "generic", - sql: "WITH h AS (SELECT @tidy(a) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", + sql: "WITH h AS (SELECT @py.tidy(a) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call macros; call macros only in __macro_actual__", None, @@ -328,7 +328,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "udf", raw: true, syntax: "generic", - sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @var('x') AS a)", + sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @py.var('x') AS a)", expected_outcome: rejected_outcome( "SQL test 'tests/t.sql' mode 'udf' CTE __udf_expected__ must not call macros", None, @@ -340,7 +340,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() { mode: "udf", raw: true, syntax: "generic", - sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@m(1)) AS b)", + sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@py.m(1)) AS b)", expected_outcome: accepted_outcome(false), }, ExtractionRuleTestCase { diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs index e2d0a816ce..b6e1ffeaff 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -580,7 +580,7 @@ fn given_compiled_sql_test_inputs_when_assembling_then_facts_match_python() { model( "order_dollars", &[], - Some("SELECT '@cents(x)', @dollars (amount)"), + Some("SELECT '@py.cents(x)', @py.dollars (amount)"), &[], ), ] diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs index 57a4708da1..810b928c46 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod aspects; pub(crate) mod reasons; +pub(crate) mod sql_macro_parts; diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs new file mode 100644 index 0000000000..9bd4f7685f --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs @@ -0,0 +1,45 @@ +//! Layout-independent text of a SQL macro's header parts. + +use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; +use crate::sql_macros::models::{ParameterKind, SqlMacroParameter}; + +/// A parameter's name, kind, types and default, independent of how the header was laid out. +pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String { + let kind: String = match ¶meter.kind { + ParameterKind::Expression { data_type } => format!("expression {}", type_text(data_type)), + ParameterKind::Constant { data_type } => format!("const {}", type_text(data_type)), + ParameterKind::Collection { set, data_type } => { + format!("collection {set} {}", type_text(data_type)) + } + ParameterKind::Enum { name } => format!("enum {}", name.to_lowercase()), + ParameterKind::Relation { columns } => relation_text(columns), + ParameterKind::ColumnOf { relation } => format!("column of {}", relation.to_lowercase()), + }; + let default: String = parameter + .default + .as_deref() + .map(sql_definition_digest) + .unwrap_or_default(); + format!("{} {kind} {default}", parameter.name.to_lowercase()) +} + +/// A relation parameter's required columns and their types. +fn relation_text(columns: &[(String, String)]) -> String { + let mut text: String = String::from("relation"); + for (column, data_type) in columns { + text.push_str(&format!( + " {} {}", + column.to_lowercase(), + type_text(data_type) + )); + } + text +} + +/// A SQL type with case and spacing ignored. +pub(crate) fn type_text(data_type: &str) -> String { + data_type + .split_whitespace() + .collect::() + .to_uppercase() +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs index 4faa4cb4ae..736bcacde4 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/mod.rs @@ -8,3 +8,4 @@ pub mod selection_json; pub mod selection_reasons; pub mod selection_report; pub mod sql_definition_digest; +pub mod sql_macro_definition_digest; diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs new file mode 100644 index 0000000000..d06562d4dc --- /dev/null +++ b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs @@ -0,0 +1,27 @@ +//! The digest of a SQL macro file's definition, with formatting, comments and description ignored. + +use crate::definition_changes::_helpers::sql_macro_parts::{parameter_text, type_text}; +use crate::definition_changes::main::definition_digest::definition_digest; +use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; +use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; + +/// The digest of SQL macro `name`'s form, arguments, result type and body; its tokens if invalid. +pub fn sql_macro_definition_digest(name: &str, path: &str, source: &str) -> String { + let Ok(definition) = parse_sql_macro(name, path, source) else { + return sql_definition_digest(source); + }; + let mut parts: Vec = vec![ + definition.form.label().to_owned(), + definition + .returns + .as_deref() + .map(type_text) + .unwrap_or_default(), + sql_definition_digest(&definition.body), + ]; + for parameter in &definition.parameters { + parts.push(parameter_text(parameter)); + } + let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect(); + definition_digest(&borrowed) +} diff --git a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs index bdb9583a5d..ac2497638f 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/tests/test_definition_changes.rs @@ -9,6 +9,7 @@ use crate::definition_changes::main::selection_json::selection_json; use crate::definition_changes::main::selection_reasons::selection_reasons; use crate::definition_changes::main::selection_report::selection_report; use crate::definition_changes::main::sql_definition_digest::sql_definition_digest; +use crate::definition_changes::main::sql_macro_definition_digest::sql_macro_definition_digest; use crate::definition_changes::models::{DefinitionChanges, SelectedItem}; use crate::definition_changes::tests::helpers::{ base_orders, current_orders, item, keys, mocking, node, renamed, snapshot, with_node, @@ -424,3 +425,50 @@ fn given_changes_and_selection_when_writing_json_then_lists_nodes_and_reasons() ); } } + +#[test] +fn given_a_sql_macro_when_fingerprinting_definition_then_ignores_layout_comments_and_description() { + let path: &str = "macros/sql/net.sql"; + let original: String = sql_macro_definition_digest( + "net", + path, + "MACRO (\n description 'Net.',\n arguments (cents INTEGER),\n returns DOUBLE,\n);\n\n\ + @cents / 100 - 1\n", + ); + let test_cases = [ + SqlDigestTestCase { + description: "header layout, a trailing comma, comments and the description are ignored", + sql: "MACRO (description 'Net after the fee.', arguments (cents integer), \ + returns double);\n\n-- one dollar fee\n@cents/100 - 1\n", + expected_same_as_original: true, + }, + SqlDigestTestCase { + description: "a body change is a change", + sql: "MACRO (description 'Net.', arguments (cents INTEGER), returns DOUBLE);\n\n\ + @cents / 100 - 2\n", + expected_same_as_original: false, + }, + SqlDigestTestCase { + description: "a parameter type change is a change", + sql: "MACRO (description 'Net.', arguments (cents BIGINT), returns DOUBLE);\n\n\ + @cents / 100 - 1\n", + expected_same_as_original: false, + }, + SqlDigestTestCase { + description: "a result type change is a change", + sql: "MACRO (description 'Net.', arguments (cents INTEGER), returns DECIMAL(18, 2));\n\n\ + @cents / 100 - 1\n", + expected_same_as_original: false, + }, + ]; + for test_case in test_cases { + let digest: String = sql_macro_definition_digest("net", path, test_case.sql); + + assert_eq!( + digest == original, + test_case.expected_same_as_original, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index af65f6af67..376fd5733b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -43,6 +43,7 @@ pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod snowflake_cost; +pub mod sql_macros; pub mod sql_test_attachment; pub mod sql_tokens; pub mod target_paths; diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index 640b985b8f..e029f9af3e 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -62,31 +62,57 @@ fn evaluate_outputs( let mut relations = Relations::new(); if let Some(schema) = schema { for table in &schema.tables { - relations.insert( - table.name.to_lowercase(), - Rc::new( - table - .columns - .iter() - .map(|column| (column.name.clone(), Some(column.data_type.clone()))) - .collect(), - ), - ); + relations.insert(table.name.to_lowercase(), Rc::new(table_outputs(table))); } } let context = TypeContext { functions, dialect, used_annotations: Cell::new(false), + untyped_source: Cell::new(false), }; let outputs = context.query(expression, &relations); (outputs, context.used_annotations.get()) } +/// Outputs only when every `FROM` and `JOIN` source of the query, CTEs included, was typed. +pub(super) fn infer_complete_outputs( + expression: &Expression, + schema: Option<&ValidationSchema>, + functions: &HashMap, + dialect: DialectType, +) -> Option { + let mut relations = Relations::new(); + if let Some(schema) = schema { + for table in &schema.tables { + relations.insert(table.name.to_lowercase(), Rc::new(table_outputs(table))); + } + } + let context = TypeContext { + functions, + dialect, + used_annotations: Cell::new(false), + untyped_source: Cell::new(false), + }; + let outputs = context.query(expression, &relations); + (!context.untyped_source.get()).then_some(outputs) +} + +/// A schema table's columns with their declared types. +fn table_outputs(table: &polyglot_sql::SchemaTable) -> Outputs { + table + .columns + .iter() + .map(|column| (column.name.clone(), Some(column.data_type.clone()))) + .collect() +} + struct TypeContext<'a> { functions: &'a HashMap, dialect: DialectType, used_annotations: Cell, + /// Whether a `FROM` or `JOIN` source could not be typed, so stars and columns may be missing. + untyped_source: Cell, } impl TypeContext<'_> { @@ -162,6 +188,15 @@ impl TypeContext<'_> { .flat_map(|from| &from.expressions) .chain(select.joins.iter().map(|join| &join.this)); for source in tables { + let typed: bool = match unwrapped(source) { + Expression::Values(_) => true, + Expression::Subquery(subquery) => subquery.alias.is_some(), + Expression::Table(table) => relations.contains_key(&table.name.name.to_lowercase()), + _ => false, + }; + if !typed { + self.untyped_source.set(true); + } if let Expression::Values(values) = unwrapped(source) { aliases.items.push(( values diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs index 3e02e0d854..2634257679 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -1,4 +1,5 @@ pub mod analyze; pub mod analyze_project; pub mod analyze_project_compact; +pub mod output_types; pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs new file mode 100644 index 0000000000..c7d83d7d89 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs @@ -0,0 +1,38 @@ +use std::collections::HashMap; + +use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; + +use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_complete_outputs}; + +/// The output columns and inferred types of the one query in `sql`; `None` if a source is untyped. +#[must_use] +pub fn output_types( + sql: &str, + dialect: &str, + schema: &ValidationSchema, +) -> Option)>> { + let dialect: DialectType = match dialect.parse() { + Ok(dialect) => dialect, + Err(_) => return None, + }; + let mut statements: Vec = + match polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()) { + Ok(statements) => statements, + Err(_) => return None, + }; + if statements.len() != 1 { + return None; + } + let expression: Expression = statements.pop()?; + let mut outputs: Vec<(String, Option)> = Vec::new(); + let inferred = infer_complete_outputs(&expression, Some(schema), &HashMap::new(), dialect)?; + for (name, data_type) in inferred { + outputs.push((name, data_type.filter(|value| !is_null_type(value)))); + } + Some(outputs) +} + +/// Whether `value` is the placeholder inference uses for a `NULL` literal's type. +fn is_null_type(value: &str) -> bool { + value == NULL_TYPE +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs index 6761b1fbd3..11698e22b1 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_function_names.rs @@ -128,7 +128,7 @@ fn given_supported_or_non_builtin_call_when_scanning_then_reports_nothing() { description: "methods, qualified functions, sentinels and macros are not checked", dialect: DialectType::Snowflake, sql: "SELECT tools.starts_with(a, 'x'), analytics.helpers.lcase(a), \ - __sqlbuild_udf_starts_with(a, 'x'), @starts_with(a), payload:lcase(a) \ + __sqlbuild_udf_starts_with(a, 'x'), @py.starts_with(a), payload:lcase(a) \ FROM __ref(\"products\")", expected_calls: &[], }, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs new file mode 100644 index 0000000000..c696813797 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs @@ -0,0 +1,619 @@ +//! A call's argument list: split, bound to parameters and checked against their types. + +use crate::sql_macros::_helpers::header::{split_top_level, unquoted}; +use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::constants::{ + ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, +}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, BoundArgument, CallArgument, ParameterKind, SqlMacroDefinition, + SqlMacroParameter, TypeFamily, +}; +use crate::type_system::main::normalize_type::normalize_type; + +const LITERAL_WORDS: [&str; 3] = ["TRUE", "FALSE", "NULL"]; +const TYPED_LITERAL_WORDS: [&str; 5] = ["DATE", "TIMESTAMP", "TIME", "INTERVAL", "TIMESTAMPTZ"]; + +/// Each top-level argument of `arguments` with its `name :=` prefix, if any. +pub(crate) fn split_arguments(arguments: &str) -> Result, SqlMacroError> { + let scanned: Vec = tokens(arguments).map_err(|_| { + SqlMacroError::new( + ARGUMENT_CODE, + "macro call arguments have an unclosed quote or comment", + ) + })?; + let mut found: Vec = Vec::new(); + for part in split_top_level(arguments, &scanned) { + let named: bool = part.len() >= NAMED_ARGUMENT_MIN_TOKENS + && part[0].kind == TokenKind::Word + && part[1].is(arguments, ":="); + let (name, value): (Option, &[Token]) = if named { + (Some(part[0].text(arguments).to_owned()), &part[2..]) + } else { + (None, part) + }; + let (Some(first), Some(last)) = (value.first(), value.last()) else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + "a named macro argument needs a value after :=", + )); + }; + found.push(CallArgument { + name, + text: arguments[first.start..last.end].to_owned(), + offset: first.start, + }); + } + Ok(found) +} + +/// Each parameter's argument: positional first, then named, then its default. +pub(crate) fn bind_arguments( + definition: &SqlMacroDefinition, + arguments: &[CallArgument], +) -> Result, SqlMacroError> { + let name: &str = &definition.name; + let mut bound: Vec> = vec![None; definition.parameters.len()]; + let mut seen_named: bool = false; + for (position, argument) in arguments.iter().enumerate() { + let index: usize = match &argument.name { + Some(argument_name) => { + seen_named = true; + definition + .parameters + .iter() + .position(|parameter| parameter.name.eq_ignore_ascii_case(argument_name)) + .ok_or_else(|| { + SqlMacroError::new( + ARGUMENT_CODE, + format!("macro {name}() has no argument {argument_name}"), + ) + .helped(format!("its arguments are {}", parameter_names(definition))) + })? + } + None => { + if seen_named { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "positional argument after a named argument in a call to macro {name}()" + ), + )); + } + if position >= definition.parameters.len() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "macro {name}() takes {} argument{}; the call passes {}", + definition.parameters.len(), + if definition.parameters.len() == 1 { + "" + } else { + "s" + }, + arguments.len() + ), + )); + } + position + } + }; + if bound[index].is_some() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {} of macro {name}() is passed twice", + definition.parameters[index].name + ), + )); + } + bound[index] = Some(BoundArgument { + parameter: index, + text: argument.text.clone(), + from_default: false, + offset: Some(argument.offset), + }); + } + let mut complete: Vec = Vec::with_capacity(bound.len()); + for (index, argument) in bound.into_iter().enumerate() { + let parameter: &SqlMacroParameter = &definition.parameters[index]; + match (argument, ¶meter.default) { + (Some(argument), _) => complete.push(argument), + (None, Some(default)) => complete.push(BoundArgument { + parameter: index, + text: default.clone(), + from_default: true, + offset: None, + }), + (None, None) => { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "call to macro {name}() is missing argument {} {}", + parameter.name, + parameter.kind.label() + ), + )); + } + } + } + Ok(complete) +} + +fn parameter_names(definition: &SqlMacroDefinition) -> String { + if definition.parameters.is_empty() { + return "none".to_owned(); + } + definition + .parameters + .iter() + .map(|parameter| parameter.name.as_str()) + .collect::>() + .join(", ") +} + +/// How an argument is written, read from its tokens alone. +pub(crate) fn argument_shape(text: &str) -> ArgumentShape { + let Ok(scanned) = tokens(text) else { + return ArgumentShape::Expression; + }; + match scanned.as_slice() { + [] => ArgumentShape::Expression, + [single] => match single.kind { + TokenKind::Number | TokenKind::Text => ArgumentShape::Literal, + TokenKind::Word + if LITERAL_WORDS + .iter() + .any(|word| single.text(text).eq_ignore_ascii_case(word)) => + { + ArgumentShape::Literal + } + TokenKind::Word | TokenKind::Quoted => ArgumentShape::Identifier { + name: single.text(text).to_owned(), + }, + _ => ArgumentShape::Expression, + }, + [at, word] if at.is(text, "@") && word.kind == TokenKind::Word && word.start == at.end => { + ArgumentShape::ParameterReference { + name: word.text(text).to_owned(), + } + } + [sign, number] if sign.is(text, "-") && number.kind == TokenKind::Number => { + ArgumentShape::Literal + } + [word, value] + if word.kind == TokenKind::Word + && value.kind == TokenKind::Text + && TYPED_LITERAL_WORDS + .iter() + .any(|typed| word.text(text).eq_ignore_ascii_case(typed)) => + { + ArgumentShape::Literal + } + [at, keyword, open, name, close] + if at.is(text, "@") + && keyword.is_word(text, "const") + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") => + { + ArgumentShape::ConstantReference { + name: unquote(name.text(text)), + } + } + [at, keyword, open, name, close, dot, member] + if at.is(text, "@") + && keyword.is_word(text, "enum") + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") + && dot.is(text, ".") + && member.kind == TokenKind::Word => + { + ArgumentShape::EnumMember { + name: unquote(name.text(text)), + member: member.text(text).to_owned(), + } + } + [first, ..] + if (first.is(text, "[") || first.is(text, "{")) + && closing_token(text, &scanned, 0) == Some(scanned.len() - 1) => + { + ArgumentShape::InlineCollection + } + [function, open, name, close] + if function.kind == TokenKind::Word + && REFERENCE_FUNCTIONS.contains(&function.text(text)) + && open.is(text, "(") + && matches!(name.kind, TokenKind::Quoted | TokenKind::Text) + && close.is(text, ")") => + { + ArgumentShape::Reference { + call: text.trim().to_owned(), + } + } + _ => ArgumentShape::Expression, + } +} + +fn unquote(text: &str) -> String { + crate::sql_macros::_helpers::header::unquoted(text) +} + +/// The type family of a SQL literal argument. +fn literal_family(text: &str) -> TypeFamily { + let trimmed: &str = text.trim(); + let upper: String = trimmed.to_ascii_uppercase(); + if upper == NULL_WORD { + return TypeFamily::Null; + } + if BOOLEAN_WORDS.contains(&upper.as_str()) { + return TypeFamily::Boolean; + } + if trimmed.starts_with('\'') || trimmed.starts_with('$') { + return TypeFamily::Text; + } + if let Some(word) = upper.split_whitespace().next() + && TYPED_LITERAL_WORDS.contains(&word) + { + return match word { + "DATE" => TypeFamily::Date, + "TIME" => TypeFamily::Time, + "INTERVAL" => TypeFamily::Unknown, + _ => TypeFamily::Timestamp, + }; + } + let digits: &str = trimmed.trim_start_matches('-').trim(); + if digits.contains(['.', 'e', 'E']) { + TypeFamily::Decimal + } else { + TypeFamily::Integer + } +} + +/// The adapter whose type rules apply and the calling macro an `@name` argument forwards from. +#[derive(Clone, Copy)] +pub(crate) struct ShapeContext<'a> { + pub(crate) adapter: &'a str, + pub(crate) caller: Option<&'a SqlMacroDefinition>, +} + +/// Whether `shape` is something `parameter` accepts; the caller checks references it resolves. +pub(crate) fn check_shape( + definition: &SqlMacroDefinition, + parameter: &SqlMacroParameter, + (argument, shape): (&str, &ArgumentShape), + context: ShapeContext<'_>, +) -> Result<(), SqlMacroError> { + let macro_name: &str = &definition.name; + let name: &str = ¶meter.name; + let shown: &str = argument.trim(); + let expects = |accepts: &str| { + SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {}: {accepts}; got {shown}", + parameter.kind.label() + ), + ) + }; + match (¶meter.kind, shape) { + (_, ArgumentShape::ParameterReference { name: forwarded }) => { + let Some(outer) = context + .caller + .and_then(|caller| caller.parameter(forwarded).map(|found| (caller, found))) + else { + return Ok(()); + }; + if forwards(&outer.1.kind, ¶meter.kind) { + Ok(()) + } else { + Err(expects(&format!( + "a value of that kind; @{forwarded} is {} argument {} of macro {}()", + article(&outer.1.kind.label()), + outer.1.name, + outer.0.name + )) + .helped(match ¶meter.kind { + ParameterKind::Constant { .. } => format!( + "declare {} CONST in macro {}() so it is known at compile time", + outer.1.name, outer.0.name + ), + _ => "forward an argument of the same kind".to_owned(), + })) + } + } + (ParameterKind::Constant { data_type }, ArgumentShape::EnumMember { .. }) => { + if fits(TypeFamily::Text, type_family(data_type, context.adapter)) { + Ok(()) + } else { + Err(expects("a literal, a scalar constant or an enum value of that type")) + } + } + (ParameterKind::Expression { data_type }, ArgumentShape::Literal) + | (ParameterKind::Constant { data_type }, ArgumentShape::Literal) => { + let actual: TypeFamily = literal_family(argument); + if fits(actual, type_family(data_type, context.adapter)) { + Ok(()) + } else { + Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {data_type}; {shown} is {}", + family_label(actual) + ), + )) + } + } + (ParameterKind::Expression { .. }, ArgumentShape::InlineCollection) => { + Err(expects("a SQL expression")) + } + (ParameterKind::Expression { .. }, _) => Ok(()), + (ParameterKind::Constant { .. }, ArgumentShape::ConstantReference { .. }) => Ok(()), + (ParameterKind::Constant { .. }, _) => Err(expects("a literal, a scalar constant or an enum value") + .helped(format!("pass a literal, such as 2, or a constant, such as @const(\"{name}\"); a CONST argument is known at compile time, so a column or expression cannot be passed"))), + ( + ParameterKind::Collection { .. }, + ArgumentShape::InlineCollection | ArgumentShape::ConstantReference { .. }, + ) => Ok(()), + (ParameterKind::Collection { .. }, _) => { + Err(expects("an inline list or set, or a list or set constant") + .helped("pass a list, such as [\"GB\", \"FR\"], or a constant, such as @const(\"supported_countries\")")) + } + (ParameterKind::Enum { name: enum_name }, ArgumentShape::EnumMember { name: given, .. }) + if given == enum_name => + { + Ok(()) + } + (ParameterKind::Enum { name: enum_name }, _) => Err(expects(&format!( + "a member of enum {enum_name}, such as @enum(\"{enum_name}\").MEMBER" + ))), + ( + ParameterKind::Relation { .. }, + ArgumentShape::Identifier { .. } | ArgumentShape::Reference { .. }, + ) => Ok(()), + (ParameterKind::Relation { .. }, _) => Err(expects( + "a CTE name, a table alias, or __ref(), __source() or __seed()", + )), + (ParameterKind::ColumnOf { .. }, ArgumentShape::Identifier { .. }) => Ok(()), + (ParameterKind::ColumnOf { relation }, _) => { + Err(expects(&format!("a column name of {relation}"))) + } + } +} + +/// Whether an argument of kind `outer`, forwarded as `@name`, fits a parameter of kind `inner`. +fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { + match inner { + ParameterKind::Expression { .. } => matches!( + outer, + ParameterKind::Expression { .. } + | ParameterKind::Constant { .. } + | ParameterKind::Enum { .. } + | ParameterKind::ColumnOf { .. } + ), + ParameterKind::Constant { .. } => { + matches!( + outer, + ParameterKind::Constant { .. } | ParameterKind::Enum { .. } + ) + } + ParameterKind::Collection { .. } => matches!(outer, ParameterKind::Collection { .. }), + ParameterKind::Enum { name } => { + matches!(outer, ParameterKind::Enum { name: outer_name } if outer_name == name) + } + ParameterKind::Relation { .. } => matches!(outer, ParameterKind::Relation { .. }), + ParameterKind::ColumnOf { .. } => matches!(outer, ParameterKind::ColumnOf { .. }), + } +} + +fn article(label: &str) -> String { + let vowel: bool = label.starts_with(['A', 'E', 'I', 'O', 'U']); + format!("{} {label}", if vowel { "an" } else { "a" }) +} + +/// A type family as messages name it. +pub(crate) fn family_label(family: TypeFamily) -> &'static str { + match family { + TypeFamily::Integer => "an integer", + TypeFamily::Decimal => "a decimal number", + TypeFamily::Float => "a floating-point number", + TypeFamily::Text => "a string", + TypeFamily::Boolean => "a boolean", + TypeFamily::Date => "a date", + TypeFamily::Timestamp => "a timestamp", + TypeFamily::Time => "a time", + TypeFamily::Null => "NULL", + TypeFamily::Unknown => "of an unknown type", + } +} + +/// Whether the SQL text is one self-delimited term that needs no parentheses around it. +pub(crate) fn is_atomic(sql: &str) -> bool { + let Ok(scanned) = tokens(sql) else { + return false; + }; + let Some(first) = scanned.first() else { + return false; + }; + if matches!( + argument_shape(sql), + ArgumentShape::ParameterReference { .. } + ) { + return true; + } + if scanned.len() == 1 { + return matches!( + first.kind, + TokenKind::Word | TokenKind::Quoted | TokenKind::Number | TokenKind::Text + ); + } + if first.is(sql, "(") { + return closing_token(sql, &scanned, 0) == Some(scanned.len() - 1); + } + if first.is_word(sql, "CASE") { + let mut depth: isize = 0; + for (index, token) in scanned.iter().enumerate() { + if token.is_word(sql, "CASE") { + depth += 1; + } else if token.is_word(sql, "END") { + depth -= 1; + if depth == 0 { + return index == scanned.len() - 1; + } + } + } + return false; + } + let mut index: usize = 0; + while index < scanned.len() { + let token: &Token = &scanned[index]; + if !matches!(token.kind, TokenKind::Word | TokenKind::Quoted) { + return false; + } + match scanned.get(index + 1) { + None => return true, + Some(next) if next.is(sql, ".") => index += 2, + Some(next) if next.is(sql, "(") => { + return closing_token(sql, &scanned, index + 1) == Some(scanned.len() - 1); + } + Some(_) => return false, + } + } + false +} + +/// The family of a declared type, normalized as schema declaration types are for the adapter. +pub(crate) fn type_family(data_type: &str, adapter: &str) -> TypeFamily { + use crate::type_system::models::TypeFamily as Normalized; + let dialect: &str = match adapter { + "motherduck" => "duckdb", + "" => "generic", + other => other, + }; + let normalized = normalize_type(data_type, dialect) + .or_else(|_| normalize_type(data_type, "generic")) + .map(|normalization| normalization.normalized.family); + match normalized { + Ok(Normalized::Integer) => TypeFamily::Integer, + Ok(Normalized::Decimal) => TypeFamily::Decimal, + Ok(Normalized::Float) => TypeFamily::Float, + Ok(Normalized::String) => TypeFamily::Text, + Ok(Normalized::Boolean) => TypeFamily::Boolean, + Ok(Normalized::Date) => TypeFamily::Date, + Ok(Normalized::Timestamp | Normalized::Datetime) => TypeFamily::Timestamp, + Ok(Normalized::Other) | Err(_) => TypeFamily::Unknown, + } +} + +/// Whether a value of type `actual` may be passed where `expected` is declared. +pub(crate) fn fits(actual: TypeFamily, expected: TypeFamily) -> bool { + use TypeFamily::{ + Boolean, Date, Decimal, Float, Integer, Null, Text, Time, Timestamp, Unknown, + }; + matches!( + (actual, expected), + (Unknown | Null, _) + | (_, Unknown) + | (Integer, Integer | Decimal | Float) + | (Decimal | Float, Decimal | Float) + | (Date, Date | Timestamp) + | (Timestamp, Timestamp) + | (Time, Time) + | (Text, Text) + | (Boolean, Boolean) + ) +} + +/// The rendered literal and type family of each value of an inline collection's inner tokens. +pub(crate) fn collection_items( + text: &str, + inner: &[Token], + invalid: &dyn Fn(&str) -> SqlMacroError, +) -> Result, SqlMacroError> { + let mut items: Vec<(String, TypeFamily)> = Vec::new(); + let mut index: usize = 0; + while index < inner.len() { + let negative: bool = inner[index].is(text, "-"); + if negative { + index += 1; + } + let value: &Token = inner + .get(index) + .ok_or_else(|| invalid("ends with a sign"))?; + items.push( + collection_item(text, value, negative) + .ok_or_else(|| invalid("holds a value that is not a literal"))?, + ); + index += 1; + match inner.get(index) { + None => {} + Some(comma) if comma.is(text, ",") => index += 1, + Some(_) => return Err(invalid("needs commas between its values")), + } + } + Ok(items) +} + +/// One collection value as a SQL literal with its type family, if it is a literal. +fn collection_item(text: &str, value: &Token, negative: bool) -> Option<(String, TypeFamily)> { + match value.kind { + TokenKind::Quoted | TokenKind::Text => Some(( + format!("'{}'", unquoted(value.text(text)).replace('\'', "''")), + TypeFamily::Text, + )), + TokenKind::Number => { + let digits: &str = value.text(text); + let family: TypeFamily = if digits.contains(['.', 'e', 'E']) { + TypeFamily::Decimal + } else { + TypeFamily::Integer + }; + Some(( + format!("{}{digits}", if negative { "-" } else { "" }), + family, + )) + } + TokenKind::Word if !negative => match value.text(text).to_ascii_lowercase().as_str() { + "true" => Some(("TRUE".to_owned(), TypeFamily::Boolean)), + "false" => Some(("FALSE".to_owned(), TypeFamily::Boolean)), + "null" => Some(("NULL".to_owned(), TypeFamily::Null)), + _ => None, + }, + _ => None, + } +} + +/// Why a collection's values do not form one list or set, if they do not. +pub(crate) fn collection_problem(items: &[(String, TypeFamily)], set: bool) -> Option { + let typed: Vec = items + .iter() + .map(|(_, family)| *family) + .filter(|family| *family != TypeFamily::Null) + .collect(); + let Some(first) = typed.first() else { + return Some("needs at least one value that is not null".to_owned()); + }; + if typed + .iter() + .any(|family| family != first && !(is_numeric(*family) && is_numeric(*first))) + { + return Some("mixes values of different types".to_owned()); + } + if !set { + return None; + } + items.iter().enumerate().find_map(|(position, (value, _))| { + repeats(&items[..position], value) + .then(|| format!("repeats {value}; a set's values must be unique")) + }) +} + +fn is_numeric(family: TypeFamily) -> bool { + matches!(family, TypeFamily::Integer | TypeFamily::Decimal) +} + +fn repeats(earlier: &[(String, TypeFamily)], value: &str) -> bool { + earlier.iter().any(|(seen, _)| seen == value) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs new file mode 100644 index 0000000000..238aca8e35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs @@ -0,0 +1,307 @@ +//! A SQL macro body parsed as SQL, its parameter references located and replaced in place. + +use polyglot_sql::expressions::{Identifier, Select}; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; + +use crate::sql_macros::_helpers::arguments::is_atomic; +use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parameters::{ + ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, + with_placeholders, +}; +use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; +use crate::sql_macros::errors::LexError; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, BodyExpansion, ParameterKind, RenderedArgument, SqlMacroDefinition, + SqlMacroForm, SqlMacroParameter, +}; + +/// Dialects tried in turn to parse a body; the first that parses it locates its references. +const PARSE_DIALECTS: [DialectType; 8] = [ + DialectType::Generic, + DialectType::DuckDB, + DialectType::Snowflake, + DialectType::BigQuery, + DialectType::PostgreSQL, + DialectType::Databricks, + DialectType::ClickHouse, + DialectType::TSQL, +]; +const EXPRESSION_PREFIX: &str = "SELECT "; +const CTE_SUFFIX: &str = "\nSELECT 1"; + +/// The body parsed as SQL. +struct ParsedBody { + select: Box