feat!: add typed SQL macros and require @py./@sql. macro calls

README.md

@@ -167,15 +167,16 @@ SELECT
   o.order_id,
   o.customer_id,
   p.amount_cents AS total_cents,
-  @cents_to_dollars("p.amount_cents") AS total_dollars
+  @py.cents_to_dollars("p.amount_cents") AS total_dollars
 FROM __ref("stg_orders") o
 JOIN __ref("stg_payments") p USING (order_id)
 ```
 
-Macros are plain Python functions that return SQL, kept next to the models that use them:
+Python macros are plain functions that return SQL, kept next to the models that use them; SQL
+macros in `macros/sql/` cover logic-free fragments with typed, checked arguments:
 
 ```python
-# models/marts/_sqlbuild/_macros/currency.py
+# models/marts/_sqlbuild/_macros/python/currency.py
 def cents_to_dollars(column: str) -> str:
     """Convert a cents integer column to a dollars decimal with two decimal places."""
     return f"ROUND(({column}) / 100.0, 2)"

benchmarks/dbt-compare/write_dbt_dense.py

@@ -43,7 +43,7 @@
 COLUMN_RE = re.compile(
     r'^\s+(\w+) \(type (\w+), nullable (true|false), description "([^"]*)"(, audits \[not_null\])?\)'
 )
-MACRO_CALL_RE = re.compile(r'@(macro_\d{3}_\d{5})\("([^"]*)"\)')
+MACRO_CALL_RE = re.compile(r'@py\.(macro_\d{3}_\d{5})\("([^"]*)"\)')
 CAST_RE = re.compile(r"CAST\(([^ ]+) AS (\w+)\) AS (\w+)")
 
 
@@ -201,7 +201,7 @@ def write_dbt(sqb: Path, out: Path, n: int) -> Path:
     base_re = re.compile(
         r'def (base_offset_\d{3})\(expression: str\) -> str:\n    return f"\(\{expression\} \+ 1\)"'
     )
-    for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/*/*.py")):
+    for macro in sorted(sqb.glob("models/*/intermediate/clean/_sqlbuild/macros/python/*/*.py")):
         text = macro.read_text()
         name, ret = macro_def_re.search(text).groups()
         offset = offset_re.search(text)[1]

crates/sqlbuild-analysis/Cargo.toml

@@ -13,6 +13,7 @@ default = []
 test-support = []
 
 [dependencies]
+memchr.workspace = true
 polyglot-sql.workspace = true
 polyglot-sql-function-catalogs.workspace = true
 rayon.workspace = true

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs

@@ -551,6 +551,7 @@ pub(crate) fn decode_outcome(
     let binding_validated: bool = decoder.flag()?;
     decoder.finish()?;
     let outcome = ModelOutcome {
+        sql_macro_findings: Vec::new(),
         analysis: ModelAnalysis {
             analysis_succeeded,
             columns,

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs

@@ -30,6 +30,7 @@ use crate::assembly::analysis_session::_helpers::mappings::{
 };
 use crate::assembly::analysis_session::_helpers::mappings::{producers, waves};
 use crate::assembly::analysis_session::constants::{UNKNOWN_NULLABILITY, UNKNOWN_TYPE};
+use crate::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql;
 use crate::assembly::analysis_session::models::{
     AnalysisSession, ColumnFact, ContractProof, DeferredCleaning, FinishedSession, LineageFacts,
     LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, Phase, PivotOutcome, PivotTables,
@@ -38,6 +39,9 @@ use crate::assembly::analysis_session::models::{
 use crate::assembly::analysis_session::types::{Pairs, Shapes};
 use crate::semantic_validation::models::DiagnosticRow;
 use crate::semantic_validation::models::ProjectCatalog;
+use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments;
+use crate::sql_macros::models::SqlMacroFinding;
+use crate::sql_macros::types::ArgumentCheckInput;
 
 /// What an internal native failure of the session names.
 const SESSION_CONTEXT: &str = "native model analysis";
@@ -279,13 +283,56 @@ impl AnalysisSession {
                 }
             };
             self.outcomes[*model] = Some(ModelOutcome {
+                sql_macro_findings: Vec::new(),
                 analysis,
                 cleaned_sql: result.cleaned_sql,
                 deferred_cleaning: None,
                 validated_schema: schema,
                 fused_binding_validated: true,
             });
         }
+        self.check_sql_macros(models)
+    }
+
+    /// Each model's SQL macro call arguments checked against its analysed types, cache hits too.
+    fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> {
+        let checked: Vec<usize> = models
+            .iter()
+            .copied()
+            .filter(|model| {
+                let request: &ModelRequest = &self.request.models[*model];
+                !request.sql_macro_checks.is_empty() && request.placeholders.is_empty()
+            })
+            .collect();
+        self.ensure_cleaned(&checked)?;
+        let mut inputs: Vec<(usize, String, Shapes)> = Vec::with_capacity(checked.len());
+        for model in checked {
+            let Some(outcome) = self.outcomes[model].as_ref() else {
+                continue;
+            };
+            let Ok(cleaned) = resolved_cleaned_sql(outcome) else {
+                continue;
+            };
+            let schema: Shapes = self.binding_schema(&self.request.models[model]);
+            inputs.push((model, cleaned.into_owned(), schema));
+        }
+        let batch: Vec<ArgumentCheckInput<'_>> = inputs
+            .iter()
+            .map(|(model, cleaned, schema)| {
+                (
+                    cleaned.as_str(),
+                    schema.as_slice(),
+                    self.request.models[*model].sql_macro_checks.as_slice(),
+                )
+            })
+            .collect();
+        let findings: Vec<Vec<SqlMacroFinding>> =
+            check_sql_macro_arguments(&self.request.dialect, &batch);
+        for ((model, _, _), found) in inputs.iter().zip(findings) {
+            if let Some(outcome) = self.outcomes[*model].as_mut() {
+                outcome.sql_macro_findings = found;
+            }
+        }
         Ok(())
     }
 

crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs

@@ -12,6 +12,7 @@ use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog;
 use crate::assembly::analysis_session::_helpers::mappings::ShapeTable;
 use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, Pairs, Shapes};
 use crate::semantic_validation::models::DiagnosticRow;
+use crate::sql_macros::models::{SqlMacroCheck, SqlMacroFinding};
 
 /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -42,6 +43,8 @@ pub struct ModelRequest {
     /// The SQL Python's dynamic pivot proof parses: placeholders replaced by their defaults.
     pub pivot_sql: String,
     pub dynamic_families: Vec<DynamicFamily>,
+    /// What the model's analysed SQL must show for its SQL macro calls' arguments to fit.
+    pub sql_macro_checks: Vec<SqlMacroCheck>,
 }
 
 /// The reference-derived fields of a `ModelRequest`.
@@ -258,6 +261,8 @@ pub struct ModelOutcome {
     pub deferred_cleaning: Option<DeferredCleaning>,
     pub validated_schema: Shapes,
     pub fused_binding_validated: bool,
+    /// The SQL macro call arguments the analysed SQL shows do not fit; never cached.
+    pub sql_macro_findings: Vec<SqlMacroFinding>,
 }
 
 /// The finished run of the session: what Python records from it.

crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs

@@ -108,6 +108,7 @@ pub(crate) fn model(name: &str, sql: &str, sources: &[&str], refs: &[&str]) -> M
         snapshot_columns: None,
         pivot_sql: sql.to_owned(),
         dynamic_families: Vec::new(),
+        sql_macro_checks: Vec::new(),
     }
 }
 

crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs

@@ -67,6 +67,8 @@ pub fn locate_binding_diagnostics(
             line,
             column,
             location,
+            help: None,
+            related: None,
         });
     }
     Ok(located)

crates/sqlbuild-analysis/src/binding_locations/models.rs

@@ -42,4 +42,15 @@ pub struct LocatedDiagnostic {
     pub line: Option<usize>,
     pub column: Option<usize>,
     pub location: Option<AuthoredSpan>,
+    pub help: Option<String>,
+    pub related: Option<RelatedSpot>,
+}
+
+/// A second authored location that adds context to a diagnostic.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RelatedSpot {
+    pub label: String,
+    pub path: String,
+    pub line: usize,
+    pub column: usize,
 }

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/authored_macro_calls.rs

@@ -47,6 +47,16 @@ fn call_open(sql: &str, at: usize) -> Option<usize> {
         return None;
     }
     let mut cursor = at + 1 + first.len_utf8();
+    for prefix in ["py.", "sql."] {
+        if sql[at + 1..].starts_with(prefix) {
+            cursor = at + 1 + prefix.len();
+            let first = char_at(sql, cursor)?;
+            if !(first == '_' || first.is_alphabetic()) {
+                return None;
+            }
+            cursor += first.len_utf8();
+        }
+    }
     while let Some(character) = char_at(sql, cursor) {
         if !is_identifier_continue(character) {
             break;

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs

@@ -1226,6 +1226,15 @@ fn macro_calls(sql: &str, syntax: &LexicalSyntax) -> Result<Vec<String>, String>
         if byte_at(sql, index) == Some(b'@')
             && let Some((name, end)) = read_identifier(sql, index + 1, syntax)
         {
+            let (name, end) =
+                if matches!(name.as_str(), "py" | "sql") && byte_at(sql, end) == Some(b'.') {
+                    match read_identifier(sql, end + 1, syntax) {
+                        Some(called) => called,
+                        None => (name, end),
+                    }
+                } else {
+                    (name, end)
+                };
             let open = space_end(sql, end);
             if byte_at(sql, open) == Some(b'(') && !DECLARATION_CALLS.contains(&name.as_str()) {
                 if !calls.contains(&name) {

crates/sqlbuild-analysis/src/compiler/tests/helpers.rs

@@ -56,7 +56,7 @@ pub(crate) fn empty_model_fixture_marker_preserves_direct_mode_validation() -> b
 pub(crate) fn expected_projection_errors_name_the_expected_cte() -> bool {
     let cases = [
         (
-            r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#,
+            r#"{"tests":[{"sql":"WITH __macro_actual__ AS (SELECT @py.status() AS status), __macro_expected__ AS (SELECT 1 + 1) SELECT 1","fileLabel":"tests/status.sql","mode":"macro"}]}"#,
             "SQL test 'tests/status.sql' must alias every non-trivial __macro_expected__ projection",
         ),
         (

crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_extraction_rules.rs

@@ -42,7 +42,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() {
             mode: "macro",
             raw: true,
             syntax: "generic",
-            sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
+            sql: "WITH my$helper AS (SELECT 1 AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
             expected_outcome: rejected_outcome(
                 "SQL test 'tests/t.sql' CTE name 'my$helper' must be an unquoted identifier of ASCII letters, digits and underscores",
                 NAME_HELP,
@@ -284,7 +284,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() {
             mode: "macro",
             raw: true,
             syntax: "generic",
-            sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
+            sql: "WITH h AS (SELECT __udf(\"f\")(1) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
             expected_outcome: rejected_outcome(
                 "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call udf; call reusable logic only in __macro_actual__",
                 None,
@@ -316,7 +316,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() {
             mode: "macro",
             raw: true,
             syntax: "generic",
-            sql: "WITH h AS (SELECT @tidy(a) AS a), __macro_actual__ AS (SELECT @m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
+            sql: "WITH h AS (SELECT @py.tidy(a) AS a), __macro_actual__ AS (SELECT @py.m(1) AS a), __macro_expected__ AS (SELECT 1 AS a)",
             expected_outcome: rejected_outcome(
                 "SQL test 'tests/t.sql' mode 'macro' helper CTE 'h' must not call macros; call macros only in __macro_actual__",
                 None,
@@ -328,7 +328,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() {
             mode: "udf",
             raw: true,
             syntax: "generic",
-            sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @var('x') AS a)",
+            sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @py.var('x') AS a)",
             expected_outcome: rejected_outcome(
                 "SQL test 'tests/t.sql' mode 'udf' CTE __udf_expected__ must not call macros",
                 None,
@@ -340,7 +340,7 @@ fn given_authored_sql_tests_when_extracting_then_each_rule_holds() {
             mode: "udf",
             raw: true,
             syntax: "generic",
-            sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@m(1)) AS b)",
+            sql: "WITH __udf_actual__ AS (SELECT __udf(\"f\")(1) AS a), __udf_expected__ AS (SELECT @enum('Status', 'PAID') AS a, @const(@py.m(1)) AS b)",
             expected_outcome: accepted_outcome(false),
         },
         ExtractionRuleTestCase {

crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs

@@ -580,7 +580,7 @@ fn given_compiled_sql_test_inputs_when_assembling_then_facts_match_python() {
             model(
                 "order_dollars",
                 &[],
-                Some("SELECT '@cents(x)', @dollars (amount)"),
+                Some("SELECT '@py.cents(x)', @py.dollars (amount)"),
                 &[],
             ),
         ]

crates/sqlbuild-analysis/src/definition_changes/_helpers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod aspects;
 pub(crate) mod reasons;
+pub(crate) mod sql_macro_parts;

crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs

@@ -0,0 +1,45 @@
+//! Layout-independent text of a SQL macro's header parts.
+
+use crate::definition_changes::main::sql_definition_digest::sql_definition_digest;
+use crate::sql_macros::models::{ParameterKind, SqlMacroParameter};
+
+/// A parameter's name, kind, types and default, independent of how the header was laid out.
+pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String {
+    let kind: String = match &parameter.kind {
+        ParameterKind::Expression { data_type } => format!("expression {}", type_text(data_type)),
+        ParameterKind::Constant { data_type } => format!("const {}", type_text(data_type)),
+        ParameterKind::Collection { set, data_type } => {
+            format!("collection {set} {}", type_text(data_type))
+        }
+        ParameterKind::Enum { name } => format!("enum {}", name.to_lowercase()),
+        ParameterKind::Relation { columns } => relation_text(columns),
+        ParameterKind::ColumnOf { relation } => format!("column of {}", relation.to_lowercase()),
+    };
+    let default: String = parameter
+        .default
+        .as_deref()
+        .map(sql_definition_digest)
+        .unwrap_or_default();
+    format!("{} {kind} {default}", parameter.name.to_lowercase())
+}
+
+/// A relation parameter's required columns and their types.
+fn relation_text(columns: &[(String, String)]) -> String {
+    let mut text: String = String::from("relation");
+    for (column, data_type) in columns {
+        text.push_str(&format!(
+            " {} {}",
+            column.to_lowercase(),
+            type_text(data_type)
+        ));
+    }
+    text
+}
+
+/// A SQL type with case and spacing ignored.
+pub(crate) fn type_text(data_type: &str) -> String {
+    data_type
+        .split_whitespace()
+        .collect::<String>()
+        .to_uppercase()
+}

crates/sqlbuild-analysis/src/definition_changes/main/mod.rs

@@ -8,3 +8,4 @@ pub mod selection_json;
 pub mod selection_reasons;
 pub mod selection_report;
 pub mod sql_definition_digest;
+pub mod sql_macro_definition_digest;

crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs

@@ -0,0 +1,27 @@
+//! The digest of a SQL macro file's definition, with formatting, comments and description ignored.
+
+use crate::definition_changes::_helpers::sql_macro_parts::{parameter_text, type_text};
+use crate::definition_changes::main::definition_digest::definition_digest;
+use crate::definition_changes::main::sql_definition_digest::sql_definition_digest;
+use crate::sql_macros::main::parse_sql_macro::parse_sql_macro;
+
+/// The digest of SQL macro `name`'s form, arguments, result type and body; its tokens if invalid.
+pub fn sql_macro_definition_digest(name: &str, path: &str, source: &str) -> String {
+    let Ok(definition) = parse_sql_macro(name, path, source) else {
+        return sql_definition_digest(source);
+    };
+    let mut parts: Vec<String> = vec![
+        definition.form.label().to_owned(),
+        definition
+            .returns
+            .as_deref()
+            .map(type_text)
+            .unwrap_or_default(),
+        sql_definition_digest(&definition.body),
+    ];
+    for parameter in &definition.parameters {
+        parts.push(parameter_text(parameter));
+    }
+    let borrowed: Vec<&str> = parts.iter().map(String::as_str).collect();
+    definition_digest(&borrowed)
+}