From d2f21de8b3a893f608d1425321f961fbf9d59aca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:12:02 +0100 Subject: [PATCH 01/28] chore: adopt the typed expression marker in Polyglot and reserve its name Polyglot lowers __polyglot_typed(expr, 'TYPE') into a typed expression node. Authored SQL may not use that name, and the analysis cache format moves to v3 so entries typed by the old post-pass are dropped. --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- .../src/assembly/analysis_session/constants.rs | 2 +- .../src/cursor_intrinsics/_helpers/mod.rs | 1 + .../src/cursor_intrinsics/_helpers/reserved.rs | 17 +++++++++++++++++ .../src/cursor_intrinsics/constants.rs | 4 ++++ .../cursor_intrinsics/main/intrinsic_free.rs | 4 ++++ .../main/validated_model_intrinsics.rs | 4 ++++ .../src/cursor_intrinsics/mod.rs | 1 + .../tests/test_intrinsic_free.rs | 7 +++++++ 10 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs create mode 100644 crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs diff --git a/Cargo.lock b/Cargo.lock index 2ab620b8a5..a755186267 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" +source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index fc1cd48a5f..41c608b4cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 2b786482a0..0cd68c4162 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -196,7 +196,7 @@ pub(crate) const PYTHON_RULE: &str = "python"; pub(crate) const NON_NULL_NULLABILITY: &str = "non_null"; pub(crate) const NULLABLE_NULLABILITY: &str = "nullable"; /// Bumped whenever the stored outcome or a key's fields change; older entries never match. -pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v2"; +pub(crate) const ANALYSIS_CACHE_FORMAT: &str = "native-model-analysis-v3"; /// A stored varint byte at or above this value continues into the next byte. pub(crate) const VARINT_CONTINUATION: u64 = 0x80; /// The value bits each stored varint byte carries. diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs index 49f9fa71a8..7309ba181d 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod reserved; pub(crate) mod scan; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs new file mode 100644 index 0000000000..5178cc2890 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/reserved.rs @@ -0,0 +1,17 @@ +//! Authored SQL that names the reserved analysis function. + +use crate::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION; + +/// The error for `context` when `sql` names the reserved analysis function in any case. +pub(crate) fn reserved_function_error(sql: &str, context: &str) -> Option { + let name: &[u8] = RESERVED_ANALYSIS_FUNCTION.as_bytes(); + sql.as_bytes() + .windows(name.len()) + .any(|window| window.eq_ignore_ascii_case(name)) + .then(|| { + format!( + "{context} uses {RESERVED_ANALYSIS_FUNCTION}, a name reserved for SQL analysis; \ + rename it" + ) + }) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs new file mode 100644 index 0000000000..c9ca3318e5 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/constants.rs @@ -0,0 +1,4 @@ +//! Names authored SQL may not use because analysis gives them a meaning of its own. + +/// The analysis-only type assertion the SQL parser lowers into a typed expression. +pub const RESERVED_ANALYSIS_FUNCTION: &str = "__polyglot_typed"; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs index c50e3bbc2f..570e757270 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs @@ -1,5 +1,6 @@ //! `reject_cursor_intrinsics`, with the error it raises. +use crate::cursor_intrinsics::_helpers::reserved::reserved_function_error; use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::cursor_intrinsics::models::IntrinsicCheck; @@ -14,6 +15,9 @@ pub fn intrinsic_free(sql: &str, reserved_markers: &[String], context: &str) -> "{context} contains a reserved internal cursor marker" )); } + if let Some(message) = reserved_function_error(sql, context) { + return IntrinsicCheck::Rejected(message); + } match replace_intrinsics(sql, context, str::to_owned) { Err(message) => IntrinsicCheck::Rejected(message), Ok((_, true)) => IntrinsicCheck::Rejected(format!( diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs index b358a56c80..c6301185ac 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs @@ -1,5 +1,6 @@ //! `get_validated_model_cursor_intrinsics`: check and canonicalize one model query's intrinsics. +use crate::cursor_intrinsics::_helpers::reserved::reserved_function_error; use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::cursor_intrinsics::models::IntrinsicModel; @@ -18,6 +19,9 @@ pub fn validated_model_intrinsics( "{context} contains a reserved internal cursor marker" )); } + if let Some(message) = reserved_function_error(sql, &context) { + return Err(message); + } let (canonical, found) = replace_intrinsics(sql, &context, |name| format!("{name}()"))?; if !found { return Ok(canonical); diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs index e6e5a52eb4..f01181f853 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs index 3fb9bd9b42..c09ecfa1ac 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs @@ -90,6 +90,13 @@ fn given_sql_when_checking_cursor_intrinsics_then_python_acceptance_or_error_is_ sql: "SELECT ยท__cursor_end", expected_check: rejected("Audit 'fresh' intrinsic __cursor_end must be called with ()"), }, + IntrinsicCheckTestCase { + description: "the reserved analysis function is rejected in any case", + sql: "SELECT __Polyglot_Typed(amount, 'BIGINT') FROM orders", + expected_check: rejected( + "Audit 'fresh' uses __polyglot_typed, a name reserved for SQL analysis; rename it", + ), + }, ]; for test_case in test_cases { From 11663a2a7313759f4790e859921092faa43ff33c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:12:03 +0100 Subject: [PATCH 02/28] feat: record where each expression SQL macro call lands in the compiled query Expansion returns the byte range of every expression macro call, top-level and nested, through body substitution, declaration rendering, Python macro splicing and cursor canonicalisation. Calls inside Python macro arguments are not recorded. --- .../src/sql_macros/_helpers/body.rs | 88 +++++++-- .../src/sql_macros/_helpers/expansion.rs | 111 ++++++++--- .../src/sql_macros/_helpers/mod.rs | 1 + .../src/sql_macros/_helpers/typed_calls.rs | 144 ++++++++++++++ .../src/sql_macros/main/byte_text_splices.rs | 9 + .../sql_macros/main/expand_sql_macro_call.rs | 6 +- .../src/sql_macros/main/mod.rs | 5 + .../src/sql_macros/main/moved_typed_calls.rs | 8 + .../sql_macros/main/spliced_typed_calls.rs | 17 ++ .../sql_macros/main/typed_analysis_pass.rs | 32 +++ .../src/sql_macros/main/typed_analysis_sql.rs | 9 + .../src/sql_macros/models.rs | 37 ++++ .../sqlbuild-analysis/src/sql_macros/tests.rs | 2 + .../src/sql_macros/tests/helpers.rs | 37 +++- .../src/sql_macros/tests/test_typed_calls.rs | 185 ++++++++++++++++++ .../src/sql_macros/tests/test_types.rs | 16 ++ .../sqlbuild-analysis/src/sql_macros/types.rs | 6 +- .../main/replace_intrinsics.rs | 20 +- crates/sqlbuild-project/src/models.rs | 4 +- crates/sqlbuild-project/src/tests/helpers.rs | 1 + .../_helpers/input_loops/model_loop.rs | 48 +++++ .../_helpers/input_loops/model_loop_host.rs | 4 + .../_helpers/render/declaration_expansion.rs | 59 +++++- .../_helpers/render/sql_macro_host.rs | 25 ++- 24 files changed, 803 insertions(+), 71 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs index 46bfffd2b0..fa16a29ae9 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs @@ -9,11 +9,12 @@ use crate::sql_macros::_helpers::parameters::{ ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, with_placeholders, }; +use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentShape, BodyExpansion, ParameterKind, RenderedArgument, SqlMacroDefinition, - SqlMacroForm, SqlMacroParameter, + ArgumentShape, BodyExpansion, ExpandedText, ParameterKind, RenderedArgument, + SqlMacroDefinition, SqlMacroForm, SqlMacroParameter, TextSplice, TypedCall, }; /// Dialects tried in turn to parse a body; the first that parses it locates its references. @@ -224,13 +225,15 @@ fn substituted( } } -/// `body` with each `@name` replaced by its argument, token by token. +/// `body` with each `@name` replaced by its argument, token by token, and the calls it holds. fn substituted_body( definition: &SqlMacroDefinition, (body, scanned, references): (&str, &[Token], &[ParameterReference]), - arguments: &[RenderedArgument], -) -> String { + (arguments, body_calls): (&[RenderedArgument], &[TypedCall]), +) -> (String, Vec) { let mut expanded: String = String::with_capacity(body.len()); + let mut splices: Vec = Vec::new(); + let mut argument_calls: Vec = Vec::new(); let mut cursor: usize = 0; for reference in references { let Some(argument) = arguments.iter().find(|argument| { @@ -242,24 +245,58 @@ fn substituted_body( }; let parameter = &definition.parameters[argument.parameter]; expanded.push_str(&body[cursor..reference.start]); - expanded.push_str(&substituted( + let text: String = substituted( ¶meter.kind, ¶meter.name, argument, (body, scanned, reference.token), - )); + ); + let start: usize = expanded.len(); + if !argument.calls.is_empty() { + let inner: Option = if text == argument.sql { + Some(0) + } else if text.len() == argument.sql.len() + 2 && text[1..].starts_with(&argument.sql) { + Some(1) + } else { + None + }; + if let Some(inner) = inner { + argument_calls.extend(shifted(argument.calls.clone(), start + inner)); + } + } + expanded.push_str(&text); + splices.push(TextSplice { + source: (reference.start, reference.end), + output: (start, expanded.len()), + }); cursor = reference.end; } expanded.push_str(&body[cursor..]); - expanded + let mut calls: Vec = argument_calls; + calls.extend(spliced(body_calls.to_vec(), &splices)); + (expanded, calls) +} + +/// `calls` in `text` kept within its `range`, relative to the range's start. +fn within(calls: Vec, (start, end): (usize, usize)) -> Vec { + calls + .into_iter() + .filter(|call| start <= call.start && call.start < end) + .map(|call| TypedCall { + start: call.start - start, + end: call.end.min(end) - start, + ..call + }) + .collect() } /// `body` with each parameter replaced by its argument, as the form inserts it at a call. pub(crate) fn expand_body( definition: &SqlMacroDefinition, - body: &str, + body: &ExpandedText, arguments: &[RenderedArgument], ) -> Result { + let (body, body_calls): (&str, &[TypedCall]) = (&body.sql, &body.calls); let invalid = |reason: String| { SqlMacroError::new( INVALID_DEFINITION_CODE, @@ -286,12 +323,27 @@ pub(crate) fn expand_body( format!("macro {}(): {reason}", definition.name), ) })?; - let expanded: String = substituted_body(definition, (body, &scanned, &references), arguments); - let sql: String = match definition.form { - SqlMacroForm::Expression => expanded.trim().to_owned(), + let (expanded, calls) = substituted_body( + definition, + (body, &scanned, &references), + (arguments, body_calls), + ); + let kept: (usize, usize) = match definition.form { + SqlMacroForm::Expression => trimmed_range(&expanded, 0), SqlMacroForm::CteList | SqlMacroForm::ColumnList => without_leading_keyword(&expanded), }; - Ok(BodyExpansion { sql, names }) + Ok(BodyExpansion { + sql: expanded[kept.0..kept.1].to_owned(), + names, + calls: within(calls, kept), + }) +} + +/// The byte range of `text[from..]` trimmed of surrounding whitespace. +fn trimmed_range(text: &str, from: usize) -> (usize, usize) { + let rest: &str = &text[from..]; + let start: usize = from + rest.len() - rest.trim_start().len(); + (start, start + rest.trim().len()) } /// `sql`, with a line break when it ends in a `--` comment and `followed` text would join that line. @@ -327,13 +379,13 @@ pub(crate) fn with_line_comment_closed(sql: &str, followed: bool) -> String { sql } -/// The text after its first token, the body's `WITH` or `SELECT`, trimmed. -fn without_leading_keyword(text: &str) -> String { +/// The byte range of the text after its first token, the body's `WITH` or `SELECT`, trimmed. +fn without_leading_keyword(text: &str) -> (usize, usize) { match tokens(text) { Ok(scanned) => match scanned.first() { - Some(first) => text[first.end..].trim().to_owned(), - None => text.trim().to_owned(), + Some(first) => trimmed_range(text, first.end), + None => trimmed_range(text, 0), }, - Err(_) => text.trim().to_owned(), + Err(_) => trimmed_range(text, 0), } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index c8e3ed4db7..cbbd08a963 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -15,6 +15,7 @@ use crate::sql_macros::_helpers::literal_positions::{ use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; use crate::sql_macros::_helpers::placement::top_level_parts; use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ ARGUMENT_CODE, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, MISPLACED_CALL_CODE, NAME_CLASH_CODE, @@ -23,8 +24,9 @@ use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallPosition, - CallScope, InsertedNames, ParameterKind, RenderedArgument, SqlMacroCall, SqlMacroCheck, - SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TextContext, TypedOutput, + CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, RenderedDeclarations, + SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TextContext, + TypedCall, TypedOutput, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -123,10 +125,13 @@ fn located(failure: ExpansionFailure, location: &SqlMacroLocation) -> Expa pub(crate) fn expand_text( host: &H, context: &TextContext<'_>, -) -> Result> { +) -> Result> { let calls: Vec = sql_macro_calls(context.text)?; if calls.is_empty() { - return host.render_declarations(context.scope, context.text); + return Ok(ExpandedText { + sql: host.render_declarations(context.scope, context.text)?.sql, + calls: Vec::new(), + }); } let scanned: Vec = tokens(context.text).map_err(|_| { SqlMacroError::new( @@ -135,17 +140,23 @@ pub(crate) fn expand_text( ) })?; let mut inserted: InsertedNames = InsertedNames::default(); - let mut spliced: String = String::with_capacity(context.text.len()); + let mut spliced_text: String = String::with_capacity(context.text.len()); + let mut typed: Vec = Vec::new(); let mut cursor: usize = 0; for call in &calls { let (expansion, updated) = expand_call(host, context, (&scanned, inserted), call)?; inserted = updated; - spliced.push_str(&context.text[cursor..call.start]); - spliced.push_str(&expansion); + spliced_text.push_str(&context.text[cursor..call.start]); + typed.extend(shifted(expansion.calls, spliced_text.len())); + spliced_text.push_str(&expansion.sql); cursor = call.replaced_end; } - spliced.push_str(&context.text[cursor..]); - host.render_declarations(context.scope, &spliced) + spliced_text.push_str(&context.text[cursor..]); + let rendered: RenderedDeclarations = host.render_declarations(context.scope, &spliced_text)?; + Ok(ExpandedText { + calls: spliced(typed, &rendered.splices), + sql: rendered.sql, + }) } /// One call's expansion as it replaces the call text in `context.text`. @@ -154,7 +165,7 @@ pub(crate) fn expand_call( context: &TextContext<'_>, caller: (&[Token], InsertedNames), call: &SqlMacroCall, -) -> Result<(String, InsertedNames), ExpansionFailure> { +) -> Result<(ExpandedText, InsertedNames), ExpansionFailure> { let (scanned, inserted) = caller; let definition: Arc = host.definition(context.scope, &call.name)?; if let Some(position) = context.stack.iter().position(|name| *name == call.name) { @@ -196,7 +207,7 @@ pub(crate) fn expand_call( } let mut stack: Vec = context.stack.to_vec(); stack.push(call.name.clone()); - let body: String = expand_text( + let body: ExpandedText = expand_text( host, &TextContext { text: &definition.body, @@ -210,7 +221,10 @@ pub(crate) fn expand_call( .map_err(|error| error.at(body_location(&definition)))?; if call.in_python { return Ok(( - python_string(&with_line_comment_closed(&expansion.sql, true)), + ExpandedText { + sql: python_string(&with_line_comment_closed(&expansion.sql, true)), + calls: Vec::new(), + }, inserted, )); } @@ -296,15 +310,19 @@ fn rendered_argument( .at(parameter_location.clone()) .into() }; - let sql: String = match (¶meter.kind, &shape) { - (ParameterKind::Collection { .. }, ArgumentShape::InlineCollection) => { - host.render_collection(text)? - } + let rendered: ExpandedText = match (¶meter.kind, &shape) { + (ParameterKind::Collection { .. }, ArgumentShape::InlineCollection) => ExpandedText { + sql: host.render_collection(text)?, + calls: Vec::new(), + }, (ParameterKind::Collection { .. }, ArgumentShape::ConstantReference { name }) => { if host.constant_shape(scope, name)? != ConstantShape::Collection { return Err(expects("a list or set constant")); } - host.render_declarations(scope, text)? + ExpandedText { + sql: host.render_declarations(scope, text)?.sql, + calls: Vec::new(), + } } ( ParameterKind::Constant { .. } | ParameterKind::Any, @@ -313,9 +331,15 @@ fn rendered_argument( if host.constant_shape(scope, name)? != ConstantShape::Scalar { return Err(expects("a scalar constant")); } - host.render_declarations(scope, text)? + ExpandedText { + sql: host.render_declarations(scope, text)?.sql, + calls: Vec::new(), + } } - (ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. }, _) => text.to_owned(), + (ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. }, _) => ExpandedText { + sql: text.to_owned(), + calls: Vec::new(), + }, _ => expand_text( host, &TextContext { @@ -328,8 +352,9 @@ fn rendered_argument( }; Ok(RenderedArgument { parameter: argument.parameter, - sql, + sql: rendered.sql, shape, + calls: rendered.calls, }) } @@ -351,13 +376,39 @@ fn python_string(sql: &str) -> String { quoted } -/// The expansion placed at the call, or why it cannot stand there or clashes. +/// The expansion placed at the call with the calls it holds, or why it cannot stand there. fn placed( definition: &SqlMacroDefinition, (site, inserted, line_continues): (&CallSite<'_>, InsertedNames, bool), expansion: &BodyExpansion, owner: &str, -) -> Result<(String, InsertedNames), SqlMacroError> { +) -> Result<(ExpandedText, InsertedNames), SqlMacroError> { + let (text, inserted, prefix) = placed_text( + definition, + (site, inserted, line_continues), + expansion, + owner, + )?; + let mut calls: Vec = shifted(expansion.calls.clone(), prefix).collect(); + if definition.form == SqlMacroForm::Expression { + let comma: usize = site.expression_separator().len(); + calls.push(TypedCall { + start: 0, + end: text.len() - comma, + macro_name: definition.name.clone(), + returns: definition.returns.clone(), + }); + } + Ok((ExpandedText { sql: text, calls }, inserted)) +} + +/// The expansion's text placed at the call, and where the expansion starts in it. +fn placed_text( + definition: &SqlMacroDefinition, + (site, inserted, line_continues): (&CallSite<'_>, InsertedNames, bool), + expansion: &BodyExpansion, + owner: &str, +) -> Result<(String, InsertedNames, usize), SqlMacroError> { let closed = |after: &str| { let followed: bool = match after.chars().next() { None => line_continues, @@ -375,9 +426,9 @@ fn placed( } let comma: &str = site.expression_separator(); if site.expression_needs_parentheses() && !is_atomic(&expansion.sql) { - Ok((format!("({}){comma}", closed(")")), inserted)) + Ok((format!("({}){comma}", closed(")")), inserted, 1)) } else { - Ok((format!("{}{comma}", closed(comma)), inserted)) + Ok((format!("{}{comma}", closed(comma)), inserted, 0)) } } SqlMacroForm::CteList => { @@ -399,7 +450,11 @@ fn placed( ("CTE", owner), )?; let (before, after) = site.cte_separators(); - Ok((format!("{before}{}{after}", closed(after)), inserted)) + Ok(( + format!("{before}{}{after}", closed(after)), + inserted, + before.len(), + )) } SqlMacroForm::ColumnList => { if position != CallPosition::SelectList { @@ -429,7 +484,11 @@ fn placed( ("column", owner), )?; let (before, after) = site.column_separators(); - Ok((format!("{before}{}{after}", closed(after)), inserted)) + Ok(( + format!("{before}{}{after}", closed(after)), + inserted, + before.len(), + )) } } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs index c7557c5132..90abead517 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs @@ -8,3 +8,4 @@ pub(crate) mod literal_positions; pub(crate) mod parameters; pub(crate) mod placement; pub(crate) mod probes; +pub(crate) mod typed_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs new file mode 100644 index 0000000000..a141eb8446 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs @@ -0,0 +1,144 @@ +//! Expression macro calls followed through the edits that move them, and their typed markers. + +use polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION; + +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::body::with_line_comment_closed; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` moved `offset` bytes later. +pub(crate) fn shifted(calls: Vec, offset: usize) -> impl Iterator { + calls.into_iter().map(move |call| TypedCall { + start: call.start + offset, + end: call.end + offset, + ..call + }) +} + +/// Where byte `position` of the source lands, or `None` inside a replaced range. +fn spliced_position(position: usize, splices: &[TextSplice]) -> Option { + let mut moved: isize = 0; + for splice in splices { + let (start, end) = splice.source; + if end <= position { + moved += (splice.output.1 - splice.output.0) as isize - (end - start) as isize; + } else if start < position { + return None; + } else { + break; + } + } + usize::try_from(position as isize + moved).ok() +} + +/// `calls` through `splices`, sorted and non-overlapping; a call a splice cuts into is dropped. +pub(crate) fn spliced(calls: Vec, splices: &[TextSplice]) -> Vec { + if splices.is_empty() { + return calls; + } + calls + .into_iter() + .filter_map(|call| { + let start: usize = spliced_position(call.start, splices)?; + let end: usize = spliced_position(call.end, splices)?; + Some(TypedCall { start, end, ..call }) + }) + .collect() +} + +/// Byte offset of each character offset of `text`, one past the end included. +fn byte_offsets(text: &str) -> Option> { + if text.is_ascii() { + return None; + } + let mut offsets: Vec = text.char_indices().map(|(offset, _)| offset).collect(); + offsets.push(text.len()); + Some(offsets) +} + +/// Character-offset `spans` from `source` to `output` as byte splices. +pub(crate) fn byte_splices(source: &str, output: &str, spans: &[SplicedSpan]) -> Vec { + let (source_offsets, output_offsets) = (byte_offsets(source), byte_offsets(output)); + let byte = |offsets: &Option>, position: usize| match offsets { + Some(offsets) => offsets.get(position).copied().unwrap_or(position), + None => position, + }; + spans + .iter() + .map(|span| TextSplice { + source: ( + byte(&source_offsets, span.source_start), + byte(&source_offsets, span.source_end), + ), + output: ( + byte(&output_offsets, span.output_start), + byte(&output_offsets, span.output_end), + ), + }) + .collect() +} + +/// One marker edit: the text inserted at a byte position, ordered among edits there. +struct MarkerEdit { + position: usize, + /// Closing edits come before opening ones at one position. + opening: bool, + /// Among openings, the outermost call first; among closings, the innermost first. + order: (usize, usize), + text: String, +} + +/// `returns` as a SQL string literal. +fn quoted(returns: &str) -> String { + format!("'{}'", returns.replace('\'', "''")) +} + +/// `sql` with each call that declares `returns` wrapped in the typed marker, and each insertion. +pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { + let mut edits: Vec = Vec::new(); + for (index, call) in calls.iter().enumerate() { + let Some(returns) = &call.returns else { + continue; + }; + let valid = |position: usize| position <= sql.len() && sql.is_char_boundary(position); + if call.start >= call.end || !valid(call.start) || !valid(call.end) { + continue; + } + let inner: &str = &sql[call.start..call.end]; + let line_break: &str = if with_line_comment_closed(inner, true).len() > inner.len() { + "\n" + } else { + "" + }; + edits.push(MarkerEdit { + position: call.start, + opening: true, + order: (usize::MAX - call.end, usize::MAX - index), + text: format!("{TYPED_EXPRESSION_FUNCTION}("), + }); + edits.push(MarkerEdit { + position: call.end, + opening: false, + order: (usize::MAX - call.start, index), + text: format!("{line_break}, {})", quoted(returns)), + }); + } + if edits.is_empty() { + return (sql.to_owned(), Vec::new()); + } + edits.sort_by_key(|edit| (edit.position, edit.opening, edit.order)); + let mut typed: String = String::with_capacity(sql.len() + edits.len() * 16); + let mut insertions: Vec = Vec::with_capacity(edits.len()); + let mut cursor: usize = 0; + for edit in edits { + typed.push_str(&sql[cursor..edit.position]); + cursor = edit.position; + insertions.push(TextSplice { + source: (edit.position, edit.position), + output: (typed.len(), typed.len() + edit.text.len()), + }); + typed.push_str(&edit.text); + } + typed.push_str(&sql[cursor..]); + (typed, insertions) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs new file mode 100644 index 0000000000..3fad5582e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs @@ -0,0 +1,9 @@ +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::typed_calls::byte_splices; +use crate::sql_macros::models::TextSplice; + +/// Character-offset `spans` from `source` to `output` as byte splices. +#[must_use] +pub fn byte_text_splices(source: &str, output: &str, spans: &[SplicedSpan]) -> Vec { + byte_splices(source, output, spans) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs index abf2e2520f..c1216cf98e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs @@ -2,16 +2,16 @@ use crate::sql_macros::_helpers::lexing::{Token, tokens}; use crate::sql_macros::constants::INVALID_DEFINITION_CODE; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; -use crate::sql_macros::models::{InsertedNames, SqlMacroCall, TextContext}; +use crate::sql_macros::models::{ExpandedText, InsertedNames, SqlMacroCall, TextContext}; use crate::sql_macros::types::SqlMacroHost; -/// One outer SQL macro call's replacement text, and the names inserted so far. +/// One outer SQL macro call's replacement text with its typed calls, and the names inserted so far. pub fn expand_sql_macro_call( host: &H, context: &TextContext<'_>, inserted: InsertedNames, call: &SqlMacroCall, -) -> Result<(String, InsertedNames), ExpansionFailure> { +) -> Result<(ExpandedText, InsertedNames), ExpansionFailure> { let scanned: Vec = tokens(context.text).map_err(|_| { SqlMacroError::new( INVALID_DEFINITION_CODE, diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index ab74c41019..08d58a97b6 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -1,9 +1,12 @@ pub mod bare_macro_calls; +pub mod byte_text_splices; pub mod check_sql_macro_arguments; pub mod declaration_visible_from_macro; pub mod expand_sql_macro_call; +pub mod moved_typed_calls; pub mod parse_sql_macro; pub mod render_inline_collection; +pub mod spliced_typed_calls; pub mod sql_macro_body_form; pub mod sql_macro_calls; pub mod sql_macro_cycle; @@ -11,4 +14,6 @@ pub mod sql_macro_declared_outputs; pub mod sql_macro_declared_type; pub mod sql_macro_output_names; pub mod sql_macro_parameter_spans; +pub mod typed_analysis_pass; +pub mod typed_analysis_sql; pub mod with_list_sql_macro_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs new file mode 100644 index 0000000000..b84e2ab843 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs @@ -0,0 +1,8 @@ +use crate::sql_macros::_helpers::typed_calls::spliced; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` moved by byte `splices`, sorted and non-overlapping; a call a splice cuts is dropped. +#[must_use] +pub fn moved_typed_calls(calls: Vec, splices: &[TextSplice]) -> Vec { + spliced(calls, splices) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs new file mode 100644 index 0000000000..c32493b136 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs @@ -0,0 +1,17 @@ +use crate::compiled_project::models::SplicedSpan; +use crate::sql_macros::_helpers::typed_calls::{byte_splices, spliced}; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `calls` in `source` moved to `output` by character-offset `spans`; a call a span cuts is dropped. +#[must_use] +pub fn spliced_typed_calls( + calls: Vec, + (source, output): (&str, &str), + spans: &[SplicedSpan], +) -> Vec { + if calls.is_empty() { + return calls; + } + let splices: Vec = byte_splices(source, output, spans); + spliced(calls, &splices) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs new file mode 100644 index 0000000000..1410c04f87 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs @@ -0,0 +1,32 @@ +use crate::semantic_validation::models::Expansion; +use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::sql_macros::models::TypedCall; + +/// `sql` with its typed calls marked, and the pass mapping the marked text's character offsets +/// back to `sql`, or `None` when no call is marked. +#[must_use] +pub fn typed_analysis_pass(sql: &str, calls: &[TypedCall]) -> Option<(String, Vec)> { + let (typed, insertions) = typed_sql(sql, calls); + if insertions.is_empty() { + return None; + } + let mut counted: ((usize, usize), (usize, usize)) = ((0, 0), (0, 0)); + let characters = |text: &str, (byte, count): &mut (usize, usize), to: usize| { + *count += text[*byte..to].chars().count(); + *byte = to; + *count + }; + let mut pass: Vec = Vec::with_capacity(insertions.len()); + for insertion in &insertions { + let source: usize = characters(sql, &mut counted.0, insertion.source.0); + let output_start: usize = characters(&typed, &mut counted.1, insertion.output.0); + let output_end: usize = characters(&typed, &mut counted.1, insertion.output.1); + pass.push(Expansion { + source_start: source, + source_end: source, + output_start, + output_end, + }); + } + Some((typed, pass)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs new file mode 100644 index 0000000000..2550414f76 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs @@ -0,0 +1,9 @@ +use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::sql_macros::models::{TextSplice, TypedCall}; + +/// `sql` with each call that declares `returns` wrapped in the analysis-only typed marker, and +/// each marker insertion as an empty source range of `sql`. +#[must_use] +pub fn typed_analysis_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { + typed_sql(sql, calls) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 8d60199c21..c4044a2b72 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -164,6 +164,8 @@ pub struct RenderedArgument { pub parameter: usize, pub sql: String, pub shape: ArgumentShape, + /// The expression macro calls `sql` holds. + pub calls: Vec, } /// One expansion's SQL and the names it defines, for clash checks at the call site. @@ -173,6 +175,41 @@ pub struct BodyExpansion { pub sql: String, /// The CTE names a CTE-list macro defines, or the column names a column-list macro outputs. pub names: Vec, + /// The expression macro calls the SQL holds, from the body and the arguments. + pub calls: Vec, +} + +/// One expression macro call's expansion in a text: its byte range and the macro's `returns`. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TypedCall { + pub start: usize, + pub end: usize, + pub macro_name: String, + /// The macro's declared result type; `None` when it declares none. + pub returns: Option, +} + +/// SQL an expansion produced and the expression macro calls it holds. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExpandedText { + pub sql: String, + /// Each expression macro call in `sql`, inner calls before the calls holding them. + pub calls: Vec, +} + +/// A replaced byte range of a text, and the byte range its replacement fills in the output. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TextSplice { + pub source: (usize, usize), + pub output: (usize, usize), +} + +/// A text with its `@const` and `@enum` references rendered, and where each one landed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RenderedDeclarations { + pub sql: String, + /// The rendered references in text order. + pub splices: Vec, } /// Where in the caller a call stands, as the text around it shows. diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index ef65d03c09..3c25706d1a 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -6,5 +6,7 @@ mod test_definitions; mod test_expansion; #[path = "tests/test_output_types.rs"] mod test_output_types; +#[path = "tests/test_typed_calls.rs"] +mod test_typed_calls; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index ec8f644996..95cb1c43d2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -5,7 +5,8 @@ use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; use crate::sql_macros::models::{ - CallScope, SqlMacroCheck, SqlMacroDefinition, TextContext, TypedOutput, + CallScope, ExpandedText, RenderedDeclarations, SqlMacroCheck, SqlMacroDefinition, TextContext, + TextSplice, TypedOutput, }; use crate::sql_macros::tests::test_types::{AsciiArguments, FakeHost}; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -53,11 +54,28 @@ impl SqlMacroHost for FakeHost { &self, _scope: &SqlMacroScope, text: &str, - ) -> Result> { - let mut rendered: String = text.to_owned(); - for (reference, value) in DECLARATIONS { - rendered = rendered.replace(reference, value); + ) -> Result> { + let mut rendered: RenderedDeclarations = RenderedDeclarations::default(); + let mut cursor: usize = 0; + while let Some((start, reference, value)) = DECLARATIONS + .iter() + .filter_map(|(reference, value)| { + text[cursor..] + .find(reference) + .map(|found| (cursor + found, *reference, *value)) + }) + .min_by_key(|(start, _, _)| *start) + { + rendered.sql.push_str(&text[cursor..start]); + let output_start: usize = rendered.sql.len(); + rendered.sql.push_str(value); + rendered.splices.push(TextSplice { + source: (start, start + reference.len()), + output: (output_start, rendered.sql.len()), + }); + cursor = start + reference.len(); } + rendered.sql.push_str(&text[cursor..]); Ok(rendered) } @@ -96,6 +114,15 @@ pub(super) fn expanded_for( macros: &[(&str, &str)], sql: &str, ) -> Result { + expanded_text_for(adapter, macros, sql).map(|expanded| expanded.sql) +} + +/// `sql` expanded with `macros` for `adapter` with its typed calls, or the error it raised. +pub(super) fn expanded_text_for( + adapter: &'static str, + macros: &[(&str, &str)], + sql: &str, +) -> Result { let host: FakeHost = FakeHost::new(macros, adapter); let context = TextContext { text: sql, diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs new file mode 100644 index 0000000000..6451c6483c --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -0,0 +1,185 @@ +use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::models::TypedCall; +use crate::sql_macros::tests::helpers::expanded_text_for; +use crate::sql_macros::tests::test_types::{TypedAnalysisTestCase, TypedCallTestCase}; + +const SHOUT: (&str, &str) = ( + "shout", + "MACRO (description \"Shout\", arguments (value VARCHAR), returns VARCHAR);\n\nUPPER(@value)\n", +); +const ADD: (&str, &str) = ( + "add", + "MACRO (description \"Add\", arguments (a BIGINT, b BIGINT), returns BIGINT);\n\n@a + @b\n", +); +const LOUD: (&str, &str) = ( + "loud", + "MACRO (description \"Loud\", arguments (value VARCHAR), returns VARCHAR);\n\n\ + @sql.shout(@value) || '!'\n", +); +const SAME: (&str, &str) = ( + "same", + "MACRO (description \"Same\", arguments (value VARCHAR), returns VARCHAR);\n\n@value\n", +); +const PLACES: (&str, &str) = ( + "places", + "MACRO (description \"Places\", arguments (value BIGINT), returns BIGINT);\n\n\ + @value + @const(\"price_places\")\n", +); + +fn shown(sql: &str, calls: &[TypedCall]) -> Vec { + calls + .iter() + .map(|call| { + format!( + "{} {}: {}", + call.macro_name, + call.returns.as_deref().unwrap_or("-"), + &sql[call.start..call.end] + ) + }) + .collect() +} + +#[test] +fn given_expression_macro_calls_when_expanding_then_each_expansion_is_recorded_where_it_lands() { + let test_cases = [ + TypedCallTestCase { + description: "a call in the select list", + macros: &[SHOUT], + sql: "SELECT o.id, @sql.shout(o.status) AS s FROM orders AS o", + expected_calls: &["shout VARCHAR: UPPER(o.status)"], + }, + TypedCallTestCase { + description: "a call parenthesised where it stands", + macros: &[ADD], + sql: "SELECT @sql.add(o.a, o.b) * 2 AS doubled FROM orders AS o", + expected_calls: &["add BIGINT: (o.a + o.b)"], + }, + TypedCallTestCase { + description: "a nested call in a body, inner first", + macros: &[SHOUT, LOUD], + sql: "SELECT @sql.loud(o.status) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "loud VARCHAR: UPPER(o.status) || '!'", + ], + }, + TypedCallTestCase { + description: "a call in another call's argument", + macros: &[SHOUT], + sql: "SELECT @sql.shout(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "shout VARCHAR: UPPER(UPPER(o.status))", + ], + }, + TypedCallTestCase { + description: "a body that is its parameter, passed another call", + macros: &[SHOUT, SAME], + sql: "SELECT @sql.same(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[ + "shout VARCHAR: UPPER(o.status)", + "same VARCHAR: UPPER(o.status)", + ], + }, + TypedCallTestCase { + description: "calls after rendered declarations", + macros: &[SHOUT, PLACES], + sql: "SELECT @const(\"price_places\") AS p, @sql.places(o.a) AS a, \ + @sql.shout(@enum(\"order_status\").OPEN) AS s FROM orders AS o", + expected_calls: &["places BIGINT: o.a + 2", "shout VARCHAR: UPPER('open')"], + }, + TypedCallTestCase { + description: "a call inside a Python macro argument is not recorded", + macros: &[SHOUT], + sql: "SELECT @py.wrap(@sql.shout(o.status)) AS s FROM orders AS o", + expected_calls: &[], + }, + ]; + for test_case in test_cases { + let expanded = expanded_text_for("duckdb", test_case.macros, test_case.sql) + .unwrap_or_else(|error| panic!("{}: {error:?}", test_case.description)); + assert_eq!( + shown(&expanded.sql, &expanded.calls), + test_case.expected_calls, + "{}: {}", + test_case.description, + expanded.sql + ); + } +} + +#[test] +fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns_is_marked() { + let test_cases = [ + TypedAnalysisTestCase { + description: "one call", + sql: "SELECT UPPER(x) AS s FROM t", + calls: &[(7, 15, Some("VARCHAR"))], + expected_sql: "SELECT __polyglot_typed(UPPER(x), 'VARCHAR') AS s FROM t", + }, + TypedAnalysisTestCase { + description: "a call without returns is not marked", + sql: "SELECT UPPER(x) AS s FROM t", + calls: &[(7, 15, None)], + expected_sql: "SELECT UPPER(x) AS s FROM t", + }, + TypedAnalysisTestCase { + description: "nested calls with one range, inner recorded first", + sql: "SELECT x FROM t", + calls: &[(7, 8, Some("INT")), (7, 8, Some("BIGINT"))], + expected_sql: "SELECT __polyglot_typed(__polyglot_typed(x, 'INT'), 'BIGINT') FROM t", + }, + TypedAnalysisTestCase { + description: "adjacent calls close before the next opens", + sql: "SELECT ab FROM t", + calls: &[(7, 8, Some("INT")), (8, 9, Some("INT"))], + expected_sql: "SELECT __polyglot_typed(a, 'INT')__polyglot_typed(b, 'INT') FROM t", + }, + TypedAnalysisTestCase { + description: "a call ending in a line comment closes it first", + sql: "SELECT x -- note\nFROM t", + calls: &[(7, 16, Some("DECIMAL(18, 2)"))], + expected_sql: "SELECT __polyglot_typed(x -- note\n, 'DECIMAL(18, 2)')\nFROM t", + }, + TypedAnalysisTestCase { + description: "a quote in returns is doubled", + sql: "SELECT x FROM t", + calls: &[(7, 8, Some("ENUM('a')"))], + expected_sql: "SELECT __polyglot_typed(x, 'ENUM(''a'')') FROM t", + }, + ]; + for test_case in test_cases { + let calls: Vec = test_case + .calls + .iter() + .map(|(start, end, returns)| TypedCall { + start: *start, + end: *end, + macro_name: "m".to_owned(), + returns: returns.map(str::to_owned), + }) + .collect(); + let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls); + assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + let removed: String = { + let mut text: String = String::new(); + let mut cursor: usize = 0; + for insertion in &insertions { + text.push_str(&typed[cursor..insertion.output.0]); + cursor = insertion.output.1; + } + text.push_str(&typed[cursor..]); + text + }; + assert_eq!(removed, test_case.sql, "{}", test_case.description); + } +} + +#[test] +fn given_the_reserved_analysis_function_when_comparing_then_authored_sql_rejects_the_parsed_name() { + assert_eq!( + sqlbuild_attachments::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION, + polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION + ); +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 5058bb8c6e..de2904e274 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -96,3 +96,19 @@ pub(super) struct MacroQuoteErrorTestCase { pub(super) expected_message: &'static str, pub(super) expected_location: (usize, usize), } + +/// A text expanded with its typed calls, each shown as `macro returns: text`. +pub(super) struct TypedCallTestCase { + pub(super) description: &'static str, + pub(super) macros: &'static [(&'static str, &'static str)], + pub(super) sql: &'static str, + pub(super) expected_calls: &'static [&'static str], +} + +/// A compiled text and its calls, with the analysis text the markers make of it. +pub(super) struct TypedAnalysisTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) calls: &'static [(usize, usize, Option<&'static str>)], + pub(super) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/types.rs b/crates/sqlbuild-analysis/src/sql_macros/types.rs index 668fe2b48a..0c6fb2ae32 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/types.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use crate::sql_macros::errors::ExpansionFailure; -use crate::sql_macros::models::{SqlMacroCheck, SqlMacroDefinition}; +use crate::sql_macros::models::{RenderedDeclarations, SqlMacroCheck, SqlMacroDefinition}; /// Whose scope a piece of SQL resolves its declarations and macros in. #[derive(Clone, Debug)] @@ -33,12 +33,12 @@ pub trait SqlMacroHost { name: &str, ) -> Result, ExpansionFailure>; - /// `text` with its `@const` and `@enum` references rendered from `scope`. + /// `text` with its `@const` and `@enum` references rendered from `scope`, and where each landed. fn render_declarations( &self, scope: &SqlMacroScope, text: &str, - ) -> Result>; + ) -> Result>; /// What the constant `name` visible from `scope` holds. fn constant_shape( diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs index 921cffd8fb..704b7d4e16 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs @@ -11,12 +11,23 @@ pub fn replace_intrinsics( context: &str, replacement: impl Fn(&str) -> String, ) -> Result<(String, bool), String> { + replace_intrinsics_with_spans(sql, context, replacement) + .map(|(replaced, spans)| (replaced, !spans.is_empty())) +} + +/// `sql` with each cursor intrinsic call replaced, and each call's byte range +/// `(source start, source end, output start, output end)`, or the malformed-call error. +pub fn replace_intrinsics_with_spans( + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, Vec<(usize, usize, usize, usize)>), String> { if !contains_intrinsic_name(sql) { - return Ok((sql.to_owned(), false)); + return Ok((sql.to_owned(), Vec::new())); } let mut replaced: String = String::with_capacity(sql.len()); let mut last: usize = 0; - let mut found: bool = false; + let mut spans: Vec<(usize, usize, usize, usize)> = Vec::new(); let mut index: usize = 0; while index < sql.len() { let (name, end): (&str, usize) = match step(sql, index) { @@ -45,11 +56,12 @@ pub fn replace_intrinsics( } }; replaced.push_str(&sql[last..index]); + let output_start: usize = replaced.len(); replaced.push_str(&replacement(name)); + spans.push((index, close + 1, output_start, replaced.len())); last = close + 1; index = close + 1; - found = true; } replaced.push_str(&sql[last..]); - Ok((replaced, found)) + Ok((replaced, spans)) } diff --git a/crates/sqlbuild-project/src/models.rs b/crates/sqlbuild-project/src/models.rs index 1c752d55df..3d6e8c124f 100644 --- a/crates/sqlbuild-project/src/models.rs +++ b/crates/sqlbuild-project/src/models.rs @@ -7,7 +7,7 @@ use sqlbuild_sqltext::compiler::models::CharSpan; use sqlbuild_analysis::assembly::analysis_session::models::{ContractProof, LineageFacts}; use sqlbuild_analysis::binding_locations::models::LocatedDiagnostic; use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_analysis::sql_macros::models::{SqlMacroCheck, TypedCall}; use sqlbuild_core::json::models::JsonValue; @@ -80,6 +80,8 @@ pub struct ModelInputRecord { pub expansion: ExpansionRecord, /// What the analysed SQL must show for the model's SQL macro call arguments to fit. pub sql_macro_checks: Vec, + /// Each expression macro call in `query_sql`, by byte range. + pub typed_calls: Vec, /// What assembly and the semantic checks finished, once the model is assembled. pub compiled: Option>, pub host: H::ModelInput, diff --git a/crates/sqlbuild-project/src/tests/helpers.rs b/crates/sqlbuild-project/src/tests/helpers.rs index e7bff79239..dfb8ee6c7e 100644 --- a/crates/sqlbuild-project/src/tests/helpers.rs +++ b/crates/sqlbuild-project/src/tests/helpers.rs @@ -48,6 +48,7 @@ pub(super) fn model_input(name: &str) -> ModelInputRecord { sql_validation_enabled: true, macro_deps: Vec::new(), sql_macro_checks: Vec::new(), + typed_calls: Vec::new(), expansion: ExpansionRecord { authored_sql: "SELECT 1".into(), expanded_sql: "SELECT 1".into(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index 149b0ea05d..a21d7440bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -14,6 +14,10 @@ use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge use sqlbuild_analysis::compiled_project::models::{ AnalysisGate, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, }; +use sqlbuild_analysis::sql_macros::main::moved_typed_calls::moved_typed_calls; +use sqlbuild_analysis::sql_macros::main::spliced_typed_calls::spliced_typed_calls; +use sqlbuild_analysis::sql_macros::models::{TextSplice, TypedCall}; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics_with_spans; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; @@ -681,6 +685,11 @@ impl NativeModelLoop { let query_sql: String = self .canonical_query(name, ¯o_sql, &values)? .map_err(|message| compile_error(context.classes, &message, None))?; + let typed_calls: Vec = query_typed_calls( + std::mem::take(&mut declarations.typed_calls), + (&declarations.spliced.sql, ¯o_sql, &query_sql), + ¯o_spans, + ); let opt_out_model: OptOutModel<'_, 'py> = OptOutModel { name, file_path, @@ -784,6 +793,7 @@ impl NativeModelLoop { analysis, named_usages, macros: macros.map(|macros| macros.output), + typed_calls, }) } @@ -974,6 +984,44 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { } /// Span rows as the spliced spans the scan-fault locator maps through. +/// The expression macro calls of the spliced SQL moved through the Python macro expansion and the +/// cursor intrinsic canonicalisation into the query's byte coordinates. +fn query_typed_calls( + calls: Vec, + (spliced, expanded, query): (&str, &str, &str), + macro_spans: &[SpanRow], +) -> Vec { + if calls.is_empty() { + return calls; + } + let calls: Vec = if macro_spans.is_empty() { + calls + } else { + spliced_typed_calls(calls, (spliced, expanded), &spliced_spans(macro_spans)) + }; + if expanded == query { + return calls; + } + let Ok((canonical, spans)) = + replace_intrinsics_with_spans(expanded, "", |name| format!("{name}()")) + else { + return Vec::new(); + }; + if canonical != query { + return Vec::new(); + } + let splices: Vec = spans + .into_iter() + .map( + |(source_start, source_end, output_start, output_end)| TextSplice { + source: (source_start, source_end), + output: (output_start, output_end), + }, + ) + .collect(); + moved_typed_calls(calls, &splices) +} + fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() .map(|row| SplicedSpan { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs index c5b50f0881..f60ed9861c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop_host.rs @@ -9,6 +9,7 @@ use sqlbuild_analysis::compiled_project::main::plain_model_facts::plain_model_fa use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, ModelConfigFacts, PlainModel, }; +use sqlbuild_analysis::sql_macros::models::TypedCall; use sqlbuild_discovery::model_files::models::HeaderFacts; use crate::bindings::_helpers::compiled_project::project::{ @@ -100,6 +101,8 @@ pub(crate) struct FinishedModel<'py> { pub(crate) named_usages: Vec>, /// The query's macro expansion, when it calls macros. pub(crate) macros: Option, + /// Each expression macro call in the query, by byte range. + pub(crate) typed_calls: Vec, } /// The Python classes a finished model's record is built with. @@ -235,6 +238,7 @@ pub(crate) fn finished_record<'py>( sql_validation_enabled: finished.analysis.0, macro_deps, sql_macro_checks: finished.declarations.sql_macro_checks.clone(), + typed_calls: finished.typed_calls.clone(), expansion: ExpansionRecord { authored: *interpolated_sql == *authored_query_sql, authored_sql: authored, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 7f3f2135aa..5428788f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -11,7 +11,7 @@ use sqlbuild_analysis::compiled_project::main::splice_declarations::splice_decla use sqlbuild_analysis::compiled_project::models::{ DeclarationSplice, InaccessibleDeclaration, SplicedSql, }; -use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; +use sqlbuild_analysis::sql_macros::models::{ExpandedText, SqlMacroCheck, TypedCall}; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::sql_macro_host::{ @@ -82,6 +82,8 @@ pub(crate) struct ExpandedDeclarations<'py> { pub(crate) sql_macros: Vec, /// What the analysed SQL must show for each top-level call's arguments to fit. pub(crate) sql_macro_checks: Vec, + /// Each expression macro call in the spliced SQL, by byte range. + pub(crate) typed_calls: Vec, } /// Splice each reference in order, raising the attachment's error for the first bad one. @@ -98,12 +100,17 @@ pub(crate) fn expand_declarations<'py>( .map(|reference| reference.start) .collect(); let (mut call_texts, sql_macros, sql_macro_checks) = if call_starts.is_empty() { - (Vec::new().into_iter(), Vec::new(), Vec::new()) + ( + Vec::::new().into_iter(), + Vec::new(), + Vec::new(), + ) } else { let (texts, facts) = expanded_sql_macro_calls(request, &call_starts)?; usages.extend(facts.usages); (texts.into_iter(), facts.names, facts.checks) }; + let mut call_splices: Vec<(usize, Vec)> = Vec::new(); for ScannedReference { kind, name, @@ -113,10 +120,14 @@ pub(crate) fn expand_declarations<'py>( } in request.references { if is_sql_macro_reference(*kind) { + let expanded: ExpandedText = call_texts.next().unwrap_or_default(); + if !expanded.calls.is_empty() { + call_splices.push((splices.len(), expanded.calls)); + } splices.push(DeclarationSplice { start: *start, end: *end, - replacement: call_texts.next().unwrap_or_default(), + replacement: expanded.sql, }); continue; } @@ -144,14 +155,54 @@ pub(crate) fn expand_declarations<'py>( if let Some(code) = request.stop { return Err(compile_error(request, &DeclarationError::ScanStop(code))); } + let spliced: SplicedSql = splice_declarations(request.sql, &splices); + let typed_calls: Vec = spliced_call_positions(&spliced, call_splices); Ok(ExpandedDeclarations { - spliced: splice_declarations(request.sql, &splices), + spliced, usages, sql_macros, sql_macro_checks, + typed_calls, }) } +/// Each call text's typed calls moved to where its splice landed in `spliced`, by byte. +fn spliced_call_positions( + spliced: &SplicedSql, + call_splices: Vec<(usize, Vec)>, +) -> Vec { + let mut typed: Vec = Vec::new(); + if call_splices.is_empty() { + return typed; + } + let ascii: bool = spliced.sql.is_ascii(); + let mut characters = spliced.sql.char_indices().map(|(offset, _)| offset); + let mut seen: usize = 0; + let mut last: usize = 0; + for (index, calls) in call_splices { + let Some(span) = spliced.spans.get(index) else { + continue; + }; + let offset: usize = if ascii { + span.output_start + } else { + if span.output_start >= seen { + last = characters + .nth(span.output_start - seen) + .unwrap_or(spliced.sql.len()); + seen = span.output_start + 1; + } + last + }; + typed.extend(calls.into_iter().map(|call| TypedCall { + start: call.start + offset, + end: call.end + offset, + ..call + })); + } + typed +} + fn enum_text<'py>( request: &DeclarationExpansion<'_, 'py>, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index b57d8924e1..2906432507 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -13,13 +13,15 @@ use sqlbuild_analysis::sql_macros::constants::{ use sqlbuild_analysis::sql_macros::errors::ExpansionFailure; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; use sqlbuild_analysis::sql_macros::main::bare_macro_calls::bare_macro_calls; +use sqlbuild_analysis::sql_macros::main::byte_text_splices::byte_text_splices; use sqlbuild_analysis::sql_macros::main::declaration_visible_from_macro::declaration_visible_from_macro; use sqlbuild_analysis::sql_macros::main::expand_sql_macro_call::expand_sql_macro_call; use sqlbuild_analysis::sql_macros::main::parse_sql_macro::parse_sql_macro; use sqlbuild_analysis::sql_macros::main::render_inline_collection::render_inline_collection; use sqlbuild_analysis::sql_macros::main::sql_macro_calls::sql_macro_calls; use sqlbuild_analysis::sql_macros::models::{ - InsertedNames, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroLocation, TextContext, + ExpandedText, InsertedNames, RenderedDeclarations, SqlMacroCall, SqlMacroCheck, + SqlMacroDefinition, SqlMacroLocation, TextContext, TextSplice, }; use sqlbuild_analysis::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -94,11 +96,12 @@ pub(crate) fn is_sql_macro_reference(kind: u8) -> bool { kind == SQL_MACRO_REFERENCE_KIND } -/// The text replacing each SQL macro call starting at the given character offsets of `request.sql`. +/// The text replacing each SQL macro call starting at the given character offsets of `request.sql`, +/// with the expression macro calls each text holds. pub(crate) fn expanded_sql_macro_calls<'py>( request: &DeclarationExpansion<'_, 'py>, starts: &[usize], -) -> PyResult<(Vec, SqlMacroFacts<'py>)> { +) -> PyResult<(Vec, SqlMacroFacts<'py>)> { let sql: &str = request.sql; let calls: Vec = sql_macro_calls(sql).map_err(|error| macro_error(request, &error))?; @@ -125,7 +128,7 @@ pub(crate) fn expanded_sql_macro_calls<'py>( owner: &owner, }; let mut inserted: InsertedNames = InsertedNames::default(); - let mut replacements: Vec = Vec::with_capacity(starts.len()); + let mut replacements: Vec = Vec::with_capacity(starts.len()); for start in starts { let Some(call) = by_char.get(start) else { return Err(macro_error( @@ -471,10 +474,13 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { &self, scope: &SqlMacroScope, text: &str, - ) -> Result> { + ) -> Result> { let (references, stop) = declaration_scan(text); if references.is_empty() && stop.is_none() { - return Ok(text.to_owned()); + return Ok(RenderedDeclarations { + sql: text.to_owned(), + splices: Vec::new(), + }); } let context: Bound<'py, PyAny> = host(self.scope_context(scope))?; if let SqlMacroScope::Macro(definition) = scope { @@ -548,7 +554,12 @@ impl<'py> SqlMacroHost for PyHost<'_, '_, 'py> { sql_macros: None, }))?; self.facts.borrow_mut().usages.extend(expanded.usages); - Ok(expanded.spliced.sql) + let splices: Vec = + byte_text_splices(text, &expanded.spliced.sql, &expanded.spliced.spans); + Ok(RenderedDeclarations { + sql: expanded.spliced.sql, + splices, + }) } fn constant_shape( From fa2cc86c90116bc897be6eb469d19ef8771efa00 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:12:03 +0100 Subject: [PATCH 03/28] feat: analyse a typed SQL macro call through an analysis-only marker The analysed text wraps each call to a macro with returns in the typed marker; the compiled SQL is unchanged. Binding diagnostics map back through the marker insertions. --- .../analysis_session/model_analyses.rs | 4 ++-- .../compile_driver/assembly_models.rs | 3 ++- .../compiled_views/compiled_model_analysis.rs | 20 ++++++++++++++---- .../compiled_views/model_input_view.rs | 21 ++++++++++--------- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 137e9a712e..e1d648e057 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compile_driver::analysis_cache::{ AnalysisCacheCounts, record_analysis_cache_metrics, }; use crate::bindings::_helpers::compiled_views::model_input_view::{ - AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, + AttachedSchema, NativeModelInput, attached_schema, input_analysis_sql, input_model_name, input_reference_rows, input_sql_macro_checks, }; use crate::bindings::_helpers::native_host::host; @@ -331,7 +331,7 @@ fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyRes let values: Bound<'py, PyAny> = native_field(model_input, "config")?.getattr("values")?; let get = |key: &str| mapping_get(&values, key); let query_sql: String = - analysis_cursor_sql(py, &input_query_sql(model_input)?, &get("cursor_type")?)?; + analysis_cursor_sql(py, &input_analysis_sql(model_input)?, &get("cursor_type")?)?; let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 5eb41c050a..3f0af799b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; use crate::bindings::_helpers::compiled_views::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_views::model_input_view::input_analysis_sql; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; use crate::bindings::_helpers::project_assembly::syntax_validation::{ @@ -80,7 +81,7 @@ fn compiled_model<'py>( py, &analysis_cursor_sql( py, - &native_field(model_input, "query_sql")?.extract::()?, + &input_analysis_sql(model_input)?, &mapping_get(&config_values, "cursor_type")?, )?, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs index 62d52b16a7..cc85d0e41d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs @@ -18,6 +18,7 @@ use sqlbuild_analysis::binding_locations::models::{ }; use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use sqlbuild_analysis::sql_macros::constants::{ARGUMENT_CODE, SUPERSEDED_ANALYSIS_CODE}; +use sqlbuild_analysis::sql_macros::main::typed_analysis_pass::typed_analysis_pass; use sqlbuild_analysis::sql_macros::models::SqlMacroFinding; use sqlbuild_project::models as project_models; @@ -165,7 +166,13 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> = vec![ + let mut typed_text: Option = None; + let typed: Option<(String, Vec)> = if *expansion.expanded_sql == *input.query_sql { + typed_analysis_pass(&expansion.expanded_sql, &input.typed_calls) + } else { + None + }; + let mut passes: Vec> = vec![ expansion .declaration_spans .iter() @@ -179,13 +186,18 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult { + passes.push(pass); + typed_text.insert(typed_sql) + } + None => &expansion.expanded_sql, + }; let source: BindingSource<'_> = BindingSource { authored, query, cleaned: &cleaned, - expansion: expansion - .authored - .then_some((&*expansion.expanded_sql, passes.as_slice())), + expansion: expansion.authored.then_some((expanded, passes.as_slice())), }; locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index ee5260e7a5..4b39a8473e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -9,6 +9,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_analysis::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use sqlbuild_analysis::sql_macros::models::SqlMacroCheck; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; @@ -548,16 +549,6 @@ pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - if let Ok(native) = model_input.cast::() { - return native.get().read(model_input.py(), |record, _| { - Ok(record.query_sql.to_string()) - }); - } - native_field(model_input, "query_sql")?.extract() -} - /// What the analysed SQL of a native model input must show for its SQL macro calls to fit. pub(crate) fn input_sql_macro_checks( model_input: &Bound<'_, PyAny>, @@ -570,6 +561,16 @@ pub(crate) fn input_sql_macro_checks( Ok(Vec::new()) } +/// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. +pub(crate) fn input_analysis_sql(model_input: &Bound<'_, PyAny>) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls).0) + }); + } + native_field(model_input, "query_sql")?.extract() +} + /// Each reference of `model_input` as `(kind, name, package, call argument count)`. pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = model_input.py(); From 9a5dc940469f2dea54ba407f5ee4ea1326c7d743 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:07:16 +0100 Subject: [PATCH 04/28] refactor: type SQL macro calls through analysis markers Replace the declared-output post-pass with the typed marker: analysis types every recorded call from its SQL and falls back to the declared return type only when inference is unknown. Typed calls flow through compiled facts and rule rows, and the explicit-cast rule accepts whole macro calls with a return type by position. --- .../_helpers/analysis_cache.rs | 16 -- .../analysis_session/_helpers/cte_facts.rs | 192 +++--------------- .../analysis_session/_helpers/dict_walk.rs | 13 +- .../analysis_session/_helpers/enrichment.rs | 2 - .../_helpers/query_columns.rs | 1 - .../analysis_session/_helpers/session.rs | 91 +-------- .../main/expression_shapes.rs | 1 - .../main/recover_cte_facts.rs | 1 - .../src/assembly/analysis_session/tests.rs | 2 + .../analysis_session/tests/helpers.rs | 26 --- .../tests/test_analysis_cache.rs | 61 +----- .../tests/test_typed_markers.rs | 103 ++++++++++ .../analysis_session/tests/test_types.rs | 10 +- .../src/compiled_project/_helpers/plain.rs | 1 + .../src/compiled_project/models.rs | 5 + .../_helpers/compatibility_types.rs | 35 +++- .../src/sql_macros/_helpers/expansion.rs | 21 +- .../src/sql_macros/_helpers/placement.rs | 116 +---------- .../src/sql_macros/constants.rs | 2 - .../src/sql_macros/main/mod.rs | 3 - .../main/sql_macro_declared_outputs.rs | 23 --- .../main/sql_macro_declared_type.rs | 40 ---- .../sql_macros/main/sql_macro_output_names.rs | 43 ---- .../src/sql_macros/models.rs | 25 --- .../sqlbuild-analysis/src/sql_macros/tests.rs | 2 - .../src/sql_macros/tests/helpers.rs | 19 +- .../src/sql_macros/tests/test_output_types.rs | 143 ------------- .../src/sql_macros/tests/test_typed_calls.rs | 11 + .../src/sql_macros/tests/test_types.rs | 21 -- .../_helpers/compiled_project/project.rs | 1 + .../_helpers/input_loops/model_loop_host.rs | 1 + .../src/engine/_helpers/project_rows.rs | 1 + .../src/engine/_helpers/retained.rs | 2 + .../src/engine/_helpers/rows.rs | 1 + .../src/engine/tests/helpers.rs | 1 + crates/sqlbuild-rules/src/models.rs | 6 + .../src/query_facts/_helpers/facts.rs | 9 + .../sqlbuild-rules/src/query_facts/models.rs | 10 + .../rules/_helpers/explicit_output_types.rs | 81 +++----- .../sql_macros/test_sql_macro_types.py | 5 +- 40 files changed, 282 insertions(+), 865 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs delete mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 129c9ec5ce..757a894a90 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -1,7 +1,5 @@ //! Per-model analysis cache keys over everything an analysis reads, and stored outcomes. -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::models::DeclaredOutput; use std::collections::{BTreeSet, HashMap}; use sha2::{Digest, Sha256}; @@ -171,20 +169,6 @@ impl KeyHasher { } self.text(&model.pivot_sql); self.families(&model.dynamic_families); - let declared: Vec = sql_macro_declared_outputs(&model.sql_macro_checks); - self.count(declared.len()); - for output in &declared { - match &output.cte { - Some(cte) => { - self.flag(true); - self.text(cte); - } - None => self.flag(false), - } - self.text(&output.name); - self.text(&output.returns); - self.count(output.branch); - } } fn families(&mut self, families: &[DynamicFamily]) { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index 74685b243c..538433175a 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -11,7 +11,7 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{ PyValue, alias_or_name, arg, children, column_table_name, direct_tables, expression_args, find_all, is_star, kind, name, name_payload, output_name, parse_one, projected_expression, py_truthy, python_int, raw_name_payload, set_operation, to_dict, top_level_ctes, - unwrap_annotations, where_clause, + typed_annotation, unwrap_annotations, where_clause, }; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::constants::{ @@ -23,7 +23,7 @@ use crate::assembly::analysis_session::constants::{ NULLABLE_NULLABILITY, NULLIF_FUNCTION_NAME, POLYGLOT_TYPE_NAMES, PYTHON_RULE, SAFE_CAST_RULE, SELECT_AST_KIND, SET_OPERATION_AST_KINDS, STRING_LITERAL_TYPE, SUBSTRING_AST_KIND, TABLE_AST_KIND, TEXT_TYPE, TIMESTAMP_TYPE_NAME, TIMESTAMP_TZ_TYPE, TRY_CAST_AST_KIND, - TYPE_PASSTHROUGH_AST_KINDS, UNKNOWN_NULLABILITY, VARCHAR_DATA_TYPES, WILDCARD, + TYPE_PASSTHROUGH_AST_KINDS, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, VARCHAR_DATA_TYPES, WILDCARD, }; use crate::assembly::analysis_session::constants::{ CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, TRANSFORM_AGGREGATION, TRANSFORM_CAST, @@ -35,8 +35,6 @@ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::models::DeclaredOutput; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeFamily}; @@ -60,8 +58,6 @@ pub(crate) struct RecoveryProfile<'a> { pub(crate) rules: Option<&'a Pairs>, /// Runs the adapter's own rules, those with the `python` rule id. pub(crate) callback: Option<&'a NullabilityCallback>, - /// The model's select items that are one SQL macro call each, typed by its `returns`. - pub(crate) declared: &'a [DeclaredOutput], } /// One enrichment's recovery request. @@ -133,108 +129,6 @@ pub(crate) struct LegacyAnalysis { pub(crate) lineage: Vec, } -/// The model query's output types its SQL macro call items change, given the relation types. -pub(crate) fn declared_output_types( - cleaned_sql: &str, - types: &ShapeTable, - profile: &RecoveryProfile<'_>, -) -> Fact> { - let Some(parsed) = parse_one(cleaned_sql, profile.dialect)? else { - return Ok(Vec::new()); - }; - let facts: Relations = referenced_types(&parsed, types); - let declared: Vec<(String, String)> = Context { - profile, - branches: declared_branches(&parsed), - } - .query_types(&parsed, &facts)?; - let plain: Vec<(String, String)> = Context { - profile, - branches: Vec::new(), - } - .query_types(&parsed, &facts)?; - let main: HashSet = main_output_columns(&parsed, profile.declared); - Ok(declared - .into_iter() - .filter(|(name, data_type)| { - dict_get(&plain, name) != Some(data_type) || main.contains(&casefold(name)) - }) - .collect()) -} - -/// Each select of the main query and top-level CTEs by address, with its scope and branch. -fn declared_branches(root: &Expression) -> Vec { - let mut branches: Vec = Vec::new(); - for (cte_name, has_column_aliases, body) in top_level_ctes(root) { - if !has_column_aliases - && unwrap_annotations(Some(body)).is_some_and(|body| top_level_ctes(body).is_empty()) - { - for (branch, select) in set_branches(Some(body)).into_iter().enumerate() { - branches.push(( - std::ptr::from_ref(select), - Some(cte_name.to_owned()), - branch, - )); - } - } - } - for (branch, select) in set_branches(Some(root)).into_iter().enumerate() { - branches.push((std::ptr::from_ref(select), None, branch)); - } - branches -} - -/// The folded main-query output columns its SQL macro call items stand in, in any branch. -fn main_output_columns(root: &Expression, declared: &[DeclaredOutput]) -> HashSet { - let selects: Vec<&Expression> = set_branches(Some(root)); - let by_name: bool = set_operation(root).is_some_and(|(_, _, by_name)| by_name); - let Some(first) = selects.first() else { - return HashSet::new(); - }; - let first_names: Vec<&str> = projection_names(first); - let mut columns: HashSet = HashSet::new(); - for output in declared.iter().filter(|output| output.cte.is_none()) { - if by_name { - columns.insert(casefold(&output.name)); - continue; - } - let Some(select) = selects.get(output.branch) else { - continue; - }; - let position: Option = projection_names(select) - .iter() - .position(|name| name.eq_ignore_ascii_case(&output.name)); - if let Some(column) = position.and_then(|position| first_names.get(position)) { - columns.insert(casefold(column)); - } - } - columns -} - -/// The output name of each of `select`'s projections, in order. -fn projection_names(select: &Expression) -> Vec<&str> { - select - .get_expressions() - .iter() - .map(|projection| output_name(unwrap_annotations(Some(projection)))) - .collect() -} - -/// The selects of `node`'s set operation, left to right, or `node` itself. -fn set_branches(node: Option<&Expression>) -> Vec<&Expression> { - let Some(node) = unwrap_annotations(node) else { - return Vec::new(); - }; - match set_operation(node) { - Some((left, right, _)) => { - let mut branches: Vec<&Expression> = set_branches(Some(left)); - branches.extend(set_branches(Some(right))); - branches - } - None => vec![node], - } -} - /// Python's legacy analysis of a model the native engine handed back, or why it defers. pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { let Some(parsed) = parse_one(input.cleaned_sql, input.profile.dialect)? else { @@ -272,7 +166,6 @@ pub(crate) fn legacy_analysis(input: &LegacyInput<'_>) -> Fact { let unqualified: Option<&(String, String)> = single_resource(&resources); let context = Context { profile: &input.profile, - branches: Vec::new(), }; let (cte_types, cte_nullability) = if input.recover { ( @@ -543,7 +436,6 @@ fn passthrough_facts( } let context = Context { profile: &input.profile, - branches: Vec::new(), }; recovery.types = context.passthrough_types(root, &input_types(input))?; recovery.nullability = context.passthrough_nullability(root, &input_nullability(input))?; @@ -574,21 +466,24 @@ fn direct_cte_outputs(root: &Expression, cte_names: &HashSet) -> HashSet outputs } -/// A select's address, its top-level CTE or `None` for the main query, and its branch. -type Branch = (*const Expression, Option, usize); - /// The profile every resolver reads. struct Context<'a, 'b> { profile: &'a RecoveryProfile<'b>, - /// The selects whose SQL macro call items take their declared types. - branches: Vec, } impl Context<'_, '_> { - /// The model query's output types, through its top-level CTEs. - fn query_types(&self, root: &Expression, facts: &Relations) -> Fact> { + /// Python's `_polyglot_cte_passthrough_types_from_parsed`. + fn passthrough_types( + &self, + root: &Expression, + facts: &Relations, + ) -> Fact> { + let ctes: Vec<(&str, bool, &Expression)> = top_level_ctes(root); + if ctes.is_empty() { + return Ok(Vec::new()); + } let mut relations: Relations = referenced_facts(root, facts); - for (cte_name, has_column_aliases, body) in top_level_ctes(root) { + for (cte_name, has_column_aliases, body) in ctes { if has_column_aliases { continue; } @@ -604,52 +499,6 @@ impl Context<'_, '_> { self.direct_select_types(Some(root), &relations) } - /// Python's `_polyglot_cte_passthrough_types_from_parsed`. - fn passthrough_types( - &self, - root: &Expression, - facts: &Relations, - ) -> Fact> { - if top_level_ctes(root).is_empty() { - return Ok(Vec::new()); - } - self.query_types(root, facts) - } - - /// `types` of `select` with each of its SQL macro call items typed by its `returns`. - fn declared_types( - &self, - select: &Expression, - mut types: Vec<(String, String)>, - ) -> Vec<(String, String)> { - let address: *const Expression = std::ptr::from_ref(select); - let Some((_, cte, branch)) = self - .branches - .iter() - .find(|(branch_select, _, _)| *branch_select == address) - else { - return types; - }; - for output in self.profile.declared { - let key: String = types - .iter() - .map(|(name, _)| name) - .find(|name| name.eq_ignore_ascii_case(&output.name)) - .cloned() - .unwrap_or_else(|| output.name.clone()); - let inferred: Option<&str> = dict_get(&types, &key).map(String::as_str); - if let Some(declared) = sql_macro_declared_type( - self.profile.declared, - (cte.as_deref(), *branch, &output.name), - inferred, - self.profile.dialect, - ) { - types = dict_set(types, &key, declared); - } - } - types - } - /// Python's `_polyglot_select_output_types`. fn select_output_types( &self, @@ -840,7 +689,7 @@ impl Context<'_, '_> { inferred = dict_set(inferred, output, inferred_type); } } - Ok(self.declared_types(select, inferred)) + Ok(inferred) } /// Python's `_polyglot_direct_expression_type`. @@ -849,6 +698,17 @@ impl Context<'_, '_> { expression: Option<&Expression>, by_alias: &Relations, ) -> Fact> { + if let Some((asserted, inner)) = typed_annotation(expression) { + if let Some(inferred) = self + .expression_type_in(Some(inner), by_alias)? + .filter(|inferred| !upper(inferred).contains(UNKNOWN_TYPE)) + { + return Ok(Some(inferred)); + } + let target: Value = + serde_json::to_value(asserted).map_err(|error| error.to_string())?; + return Ok(target.as_object().and_then(cast_target_type)); + } let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); if let Some(inferred) = self.expression_type(expression)? { @@ -1070,6 +930,7 @@ impl Context<'_, '_> { aliases: &[(String, &'static str)], scoped: &NullRelations, ) -> Fact<&'static str> { + let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); match node_kind { NULL_AST_KIND => return Ok(NULLABLE_NULLABILITY), @@ -1101,6 +962,7 @@ impl Context<'_, '_> { /// Python's `_infer_polyglot_shallow_nullability`. fn shallow_nullability(&self, expression: Option<&Expression>) -> Fact<&'static str> { + let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); match node_kind { NULL_AST_KIND => return Ok(NULLABLE_NULLABILITY), diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs index 287eebd81f..df9020df31 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs @@ -2,7 +2,7 @@ use polyglot_sql::expressions::Cte; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions, ast_json}; +use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions, ast_json}; use serde_json::{Map, Value, json}; use crate::assembly::analysis_session::models::ProjectionFacts; @@ -254,6 +254,17 @@ pub(crate) fn unwrap_annotations(mut node: Option<&Expression>) -> Option<&Expre node } +/// The first type assertion around `node`, past plain annotations, and the expression it types. +pub(crate) fn typed_annotation(mut node: Option<&Expression>) -> Option<(&DataType, &Expression)> { + while let Some(Expression::Annotated(annotated)) = node { + if let Some(asserted) = &annotated.type_assertion { + return Some((asserted, &annotated.this)); + } + node = Some(&annotated.this); + } + None +} + /// A projection's expression: an alias's `this`, otherwise the projection. pub(crate) fn projected_expression(projection: Option<&Expression>) -> Option<&Expression> { if kind(projection) == ALIAS_AST_KIND { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index cefd99d2b9..2d69ab47a5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -113,7 +113,6 @@ fn legacy(input: &EnrichmentInput<'_>, cleaned: &str) -> Result = input diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs index f360b6b503..7a20a27572 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/query_columns.rs @@ -182,7 +182,6 @@ impl Tables<'_> { function_return_types: &self.request.function_return_types, rules: self.request.nullability_rules.as_ref(), callback: self.request.nullability_callback.as_ref(), - declared: &[], }, }) })?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index d8d1e7d87a..69a0f8c235 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -19,7 +19,7 @@ use crate::assembly::analysis_session::_helpers::compact_batch::{ Batch, BatchMember, MemberAnalysis, MemberResult, }; use crate::assembly::analysis_session::_helpers::cte_facts::{ - LegacyAnalysis, LegacyInput, RecoveryProfile, declared_output_types, legacy_analysis, + LegacyAnalysis, LegacyInput, RecoveryProfile, legacy_analysis, }; use crate::assembly::analysis_session::_helpers::dynamic_pivot::{PivotFacts, pivot_outcome}; use crate::assembly::analysis_session::_helpers::enrichment::{ @@ -40,9 +40,7 @@ use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; use crate::sql_macros::main::check_sql_macro_arguments::check_sql_macro_arguments; -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::models::{DeclaredOutput, SqlMacroCheck, SqlMacroFinding}; +use crate::sql_macros::models::SqlMacroFinding; use crate::sql_macros::types::ArgumentCheckInput; /// What an internal native failure of the session names. @@ -296,76 +294,6 @@ impl AnalysisSession { self.check_sql_macros(models) } - /// Type the outputs SQL macro call items reach by the macros' `returns`, through CTEs. - fn type_sql_macro_outputs(&mut self, models: &[usize]) -> Result<(), String> { - for model in models { - let declared: Vec = - sql_macro_declared_outputs(&self.request.models[*model].sql_macro_checks); - if declared.is_empty() || self.cache_hit(*model) { - continue; - } - self.ensure_cleaned(&[*model])?; - let Some(outcome) = self.outcomes[*model].as_ref() else { - continue; - }; - let Ok(cleaned) = resolved_cleaned_sql(outcome) else { - continue; - }; - let profile = RecoveryProfile { - dialect: &self.request.dialect, - function_return_types: &self.request.function_return_types, - rules: self.request.nullability_rules.as_ref(), - callback: self.request.nullability_callback.as_ref(), - declared: &declared, - }; - let propagated: Result, String> = catch_compiler_panic(|| { - Ok(declared_output_types( - &cleaned, - &self.available_types, - &profile, - )) - }) - .map_err(|reason| { - native_failure(&model_context(&self.request.models[*model]), &reason) - })?; - let Ok(typed) = propagated else { - continue; - }; - let dialect: String = self.request.dialect.clone(); - let Some(columns) = self.outcome_mut(*model)?.analysis.columns.as_mut() else { - continue; - }; - for column in columns { - let Some((name, returns)) = typed - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case(&column.name)) - else { - continue; - }; - let reached: [DeclaredOutput; 1] = [DeclaredOutput { - cte: None, - name: name.clone(), - returns: returns.clone(), - branch: 0, - }]; - if let Some(data_type) = sql_macro_declared_type( - &reached, - (None, 0, &column.name), - column.data_type.as_deref(), - &dialect, - ) { - column.data_type = Some(data_type); - } - } - } - Ok(()) - } - - /// Whether the cache answered `model` with an outcome that already holds its declared types. - fn cache_hit(&self, model: usize) -> bool { - self.cache.as_ref().is_some_and(|cache| cache.hits[model]) - } - /// Each model's SQL macro call arguments checked against its analysed types, cache hits too. fn check_sql_macros(&mut self, models: &[usize]) -> Result<(), String> { let checked: Vec = models @@ -373,7 +301,7 @@ impl AnalysisSession { .copied() .filter(|model| { let request: &ModelRequest = &self.request.models[*model]; - checks_arguments(&request.sql_macro_checks) && request.placeholders.is_empty() + !request.sql_macro_checks.is_empty() && request.placeholders.is_empty() }) .collect(); self.ensure_cleaned(&checked)?; @@ -427,7 +355,6 @@ impl AnalysisSession { function_return_types: &self.request.function_return_types, rules: self.request.nullability_rules.as_ref(), callback: self.request.nullability_callback.as_ref(), - declared: &[], }, }) }) @@ -459,7 +386,11 @@ impl AnalysisSession { } let request: &ModelRequest = &self.request.models[model]; let required: Vec = request.required_names.clone(); - let has_set_operation: bool = request.has_set_operation; + // A set operation is enriched only to type its SQL macro calls by their markers. + let has_set_operation: bool = request.has_set_operation + && !request + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); let inputs_known: bool = !required.is_empty() && required .iter() @@ -549,7 +480,6 @@ impl AnalysisSession { /// Python's deferred binding validation, then the dataflow's shape publication. fn finish_wave(&mut self, wave: usize) -> Result<(), String> { let models: Vec = self.waves[wave].clone(); - self.type_sql_macro_outputs(&models)?; let validated: &[usize] = if self.cyclic { &[] } else { &models }; let mut revalidated: Vec<(usize, Shapes)> = Vec::new(); for model in validated { @@ -795,11 +725,6 @@ impl AnalysisSession { } } -/// Whether any of the checks has arguments the analysed SQL must show fit. -fn checks_arguments(checks: &[SqlMacroCheck]) -> bool { - checks.iter().any(|check| !check.arguments.is_empty()) -} - fn batch_member<'a>(model: &'a ModelRequest, schema: &'a Shapes) -> BatchMember<'a> { BatchMember { query_sql: &model.query_sql, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs index 27a9755d87..b65c4a0ad7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/expression_shapes.rs @@ -66,7 +66,6 @@ pub fn expression_shapes( function_return_types: &request.function_return_types, rules: request.nullability_rules.as_ref(), callback: request.nullability_callback.as_ref(), - declared: &[], }, }) })?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs index acd8cc734a..92b9f05478 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/recover_cte_facts.rs @@ -29,7 +29,6 @@ fn recovered_facts(request: &CteFactRequest) -> Result { function_return_types: &request.function_return_types, rules: request.nullability_rules.as_ref(), callback: None, - declared: &[], }, })?; let mut direct_outputs: Vec = recovered.direct_outputs.into_iter().collect(); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index 95de9dfe5c..6cb3eb54b3 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -22,3 +22,5 @@ mod native_dialect; mod reference_facts; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_typed_markers.rs"] +mod typed_markers; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 49eb329990..459254d0f7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -27,7 +27,6 @@ use crate::assembly::analysis_session::tests::test_types::{ }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::models::{CatalogInput, ProjectCatalog}; -use crate::sql_macros::models::{CallScope, SqlMacroCheck, TypedOutput}; const TRANSFORM_NAMES: [&str; 6] = [ "direct", @@ -180,11 +179,6 @@ pub(crate) fn cached_run(request: SessionRequest, store: NativeStore) -> CachedR (keys, hits, taken) }); CachedRun { - deferred: outcome - .models - .iter() - .map(|model| model.deferred_cleaning.is_some()) - .collect(), lines: ( steps.iter().map(step_lines).collect(), outcome.models.iter().map(described).collect(), @@ -477,7 +471,6 @@ pub(crate) fn recovered_facts(sql: &str) -> Option<(Pairs, Pairs, Vec, V function_return_types: &Vec::new(), rules: Some(&rules), callback: None, - declared: &[], }, }) .ok()?; @@ -557,7 +550,6 @@ pub(crate) fn legacy_lines(sql: &str) -> Option> { function_return_types: &Vec::new(), rules: Some(&rules), callback: None, - declared: &[], }, }) .ok()?; @@ -705,21 +697,3 @@ pub(crate) fn injected_relations( request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); request } - -/// The orders request over `models`, the first typing `label` in CTE `final` as VARCHAR. -pub(crate) fn declared_label_request(models: &[ModelSpec]) -> SessionRequest { - let mut requests: Vec = model_requests(models); - requests[0].sql_macro_checks.push(SqlMacroCheck { - macro_name: "label".to_owned(), - scope: CallScope::Cte("final".to_owned()), - plain_select: true, - arguments: Vec::new(), - call: None, - output: Some(TypedOutput { - name: "label".to_owned(), - returns: "VARCHAR".to_owned(), - branch: 0, - }), - }); - orders_request(requests) -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs index 914908700f..24c55a4b51 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_cache.rs @@ -3,13 +3,13 @@ use sqlbuild_cache::store::models::NativeStore; use crate::assembly::analysis_session::models::{ModelReference, SessionRequest}; use crate::assembly::analysis_session::tests::helpers::{ - amounts_family, cache_stats, cached_run, completed, current_model_key, damaged, - declared_label_request, described, model_keys, model_requests, orders_request, pairs, - second_model, session_lines, shapes, started, step_lines, uncached_catalog, + amounts_family, cache_stats, cached_run, current_model_key, damaged, model_keys, + model_requests, orders_request, pairs, second_model, session_lines, shapes, started, + uncached_catalog, }; use crate::assembly::analysis_session::tests::test_types::{ CacheEditTestCase, CacheKeyTestCase, CacheReuseTestCase, CachedRun, DamagedCacheTestCase, - DeclaredHitTestCase, ModelSpec, RelationDigestTestCase, + ModelSpec, RelationDigestTestCase, }; const STG_ORDERS: ModelSpec = ( @@ -73,15 +73,6 @@ const EVENTS_TOO_DEEP: ModelSpec = ( &[], ); -/// Labels orders with a function the inference does not type, in CTE `final`. -const LABELLED: ModelSpec = ( - "labelled", - "WITH final AS (SELECT order_id, bar(amount, 0, 10) AS label FROM __source(\"raw_orders\")) \ - SELECT * FROM final", - &["raw_orders"], - &[], -); - #[test] fn given_a_filled_cache_when_rerunning_then_hits_match_the_uncached_session() { let test_cases = [ @@ -528,47 +519,3 @@ fn given_a_read_relation_fact_changed_when_keying_then_the_reading_model_key_cha ); } } - -#[test] -fn given_a_declared_sql_macro_output_when_rerunning_then_the_hit_keeps_its_type_unpatched() { - let test_cases = [DeclaredHitTestCase { - description: "a call item in a final CTE, read with a star", - models: &[LABELLED], - expected_label: "label VARCHAR", - }]; - for test_case in test_cases { - let (uncached_steps, uncached_models, _) = - completed(started(declared_label_request(test_case.models))); - let cold: CachedRun = cached_run( - declared_label_request(test_case.models), - NativeStore::default(), - ); - let cold_lines = cold.lines; - let warm: CachedRun = cached_run(declared_label_request(test_case.models), cold.store); - let uncached = ( - uncached_steps.iter().map(step_lines).collect::>(), - uncached_models.iter().map(described).collect::>(), - ); - - assert!( - uncached.1[0] - .iter() - .any(|line| line.starts_with(test_case.expected_label)), - "{}: {:?}", - test_case.description, - uncached.1 - ); - assert_eq!( - (&cold_lines, &warm.lines), - (&uncached, &uncached), - "{}", - test_case.description - ); - assert_eq!( - (warm.hits, warm.deferred), - (vec![true], vec![true]), - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs new file mode 100644 index 0000000000..81053e52bb --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs @@ -0,0 +1,103 @@ +use sqlbuild_cache::store::models::NativeStore; + +use crate::assembly::analysis_session::models::SessionRequest; +use crate::assembly::analysis_session::tests::helpers::{ + cached_run, completed, described, model_requests, orders_request, shapes, started, +}; +use crate::assembly::analysis_session::tests::test_types::{ + CachedRun, ModelSpec, TypedMarkerTestCase, +}; + +const STAGED: ModelSpec = ( + "staged", + "SELECT * FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], +); + +/// The orders request over `sql` reading `staged`, as a fast-lineage compile of a contract sends it. +fn request(sql: &'static str) -> SessionRequest { + let mut requests = model_requests(&[("typed", sql, &[], &["staged"]), STAGED]); + requests[0].recover_cte_facts = true; + let mut request: SessionRequest = orders_request(requests); + request.rich_type_inference = false; + let own = shapes(&[("typed", &[("v", "VARCHAR")])]); + request.column_types.extend(own.clone()); + request.complete_schemas.extend(own); + request +} + +/// The described line of the first model's column `v`. +fn column_v(lines: &[Vec]) -> String { + lines[0] + .iter() + .find(|line| line.starts_with("v ")) + .cloned() + .unwrap_or_else(|| format!("no v in {:?}", lines[0])) +} + +const CASES: [TypedMarkerTestCase; 6] = [ + TypedMarkerTestCase { + description: "an unknown body takes the asserted type", + sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a known body keeps its inferred type", + sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v INT ", + }, + TypedMarkerTestCase { + description: "a later set-operation branch types the union", + sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ + SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a NULL body takes the asserted type in a union", + sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"staged\") AS o UNION ALL \ + SELECT __polyglot_typed(NULL, 'BIGINT') AS v FROM __ref(\"staged\") AS o", + expected_column: "v BIGINT", + }, + TypedMarkerTestCase { + description: "a CTE read by name", + sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ + FROM __ref(\"staged\")) SELECT v FROM c", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "an operand of a larger expression", + sql: "SELECT COALESCE(__polyglot_typed(bar(amount, 0, 10), 'VARCHAR'), 'x') AS v \ + FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, +]; + +#[test] +fn given_typed_markers_when_analysing_then_each_call_has_its_asserted_or_inferred_type() { + for test_case in CASES { + let (_, outcomes, _) = completed(started(request(test_case.sql))); + let lines: Vec> = outcomes.iter().map(described).collect(); + assert!( + column_v(&lines).starts_with(test_case.expected_column), + "{}: {}", + test_case.description, + column_v(&lines) + ); + } +} + +#[test] +fn given_typed_markers_when_rerunning_from_the_cache_then_the_hit_matches_the_cold_run() { + for test_case in CASES { + let cold: CachedRun = cached_run(request(test_case.sql), NativeStore::default()); + let warm: CachedRun = cached_run(request(test_case.sql), cold.store); + assert_eq!(warm.lines, cold.lines, "{}", test_case.description); + assert!( + warm.hits.iter().all(|hit| *hit), + "{}: {:?}", + test_case.description, + warm.hits + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index 9568fadd92..e864a71cca 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -144,8 +144,6 @@ pub(crate) struct CachedRun { pub(crate) keys: Vec>, /// Whether each model's outcome came from the cache. pub(crate) hits: Vec, - /// Whether each model's outcome still defers its cleaned SQL. - pub(crate) deferred: Vec, /// The catalog's schema additions and analysis names, then the sorted fact models. pub(crate) catalog: ( crate::assembly::analysis_session::types::Shapes, @@ -220,9 +218,9 @@ pub(crate) struct DeltaSessionTestCase { pub(crate) expected_whole_session_outcome: bool, } -/// Models whose first one declares a SQL macro output, and that output's described column. -pub(crate) struct DeclaredHitTestCase { +/// A model query holding typed markers, and the described line of its column `v`. +pub(crate) struct TypedMarkerTestCase { pub(crate) description: &'static str, - pub(crate) models: &'static [ModelSpec], - pub(crate) expected_label: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_column: &'static str, } diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs index e0f79d49e6..3f6e1724f9 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs @@ -39,5 +39,6 @@ pub(crate) fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { enum_declarations: model.enum_declarations, constant_declarations: model.constant_declarations, analysis: None, + typed_calls: model.typed_calls, } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index d769f01a51..309221dca8 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -1,5 +1,6 @@ //! Facts a compile's producers retain about each resource, in production order. +use crate::sql_macros::models::TypedCall; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; @@ -31,6 +32,8 @@ pub struct CompiledModelFacts { pub constant_declarations: Vec, /// The typed analysis result; `None` until the assembly stage transfers it. pub analysis: Option, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A model's effective config values with sorted keys, as the rules request has always read them. @@ -212,6 +215,8 @@ pub struct PlainModel { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A one-based line and column, counted in characters. diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index e029f9af3e..73e7555a7c 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -280,7 +280,9 @@ impl TypeContext<'_> { } fn projection_type(&self, expression: &Expression, aliases: &Sources) -> Option { - if matches!(unwrapped(expression), Expression::Null(_)) { + if asserted_expression(expression).is_none() + && matches!(unwrapped(expression), Expression::Null(_)) + { Some(NULL_TYPE.to_string()) } else { self.expression(expression, aliases).or_else(|| { @@ -294,6 +296,15 @@ impl TypeContext<'_> { } fn expression(&self, expression: &Expression, aliases: &Sources) -> Option { + if let Some((asserted, inner)) = asserted_expression_type(expression) { + if matches!(unwrapped(inner), Expression::Null(_)) { + return self.cast_type(asserted); + } + // An unknown inferred type leaves the assertion to the annotation fallback. + return self + .expression(inner, aliases) + .filter(|inferred| !inferred.to_ascii_uppercase().contains("UNKNOWN")); + } let expression = unwrapped(expression); let function_name = match expression { Expression::Function(function) => function.name.as_str(), @@ -435,6 +446,28 @@ fn query_with(expression: &Expression) -> Option<&With> { } } +/// The expression a type assertion around `expression` types, past parentheses and comments. +fn asserted_expression(expression: &Expression) -> Option<&Expression> { + asserted_expression_type(expression).map(|(_, inner)| inner) +} + +/// The type assertion around `expression`, past parentheses and comments, and what it types. +fn asserted_expression_type(mut expression: &Expression) -> Option<(&DataType, &Expression)> { + loop { + expression = match expression { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => { + return annotation + .type_assertion + .as_ref() + .map(|asserted| (asserted, &annotation.this)); + } + Expression::Annotated(annotation) => &annotation.this, + Expression::Paren(paren) => &paren.this, + _ => return None, + }; + } +} + fn unwrapped(mut expression: &Expression) -> &Expression { loop { expression = match expression { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index cbbd08a963..e76d4ac5d5 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -26,7 +26,7 @@ use crate::sql_macros::models::{ ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallPosition, CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, RenderedDeclarations, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TextContext, - TypedCall, TypedOutput, + TypedCall, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -235,26 +235,14 @@ pub(crate) fn expand_call( call.replaced_end, ); if matches!(context.scope, SqlMacroScope::Caller) && context.stack.is_empty() { - let output: Option = match (definition.form, &definition.returns) { - (SqlMacroForm::Expression, Some(returns)) => site - .whole_item_alias() - .zip(call_branch(context.text, scanned, call.start)) - .map(|(name, branch)| TypedOutput { - name, - returns: returns.clone(), - branch, - }), - _ => None, - }; let check: SqlMacroCheck = SqlMacroCheck { macro_name: definition.name.clone(), scope: call_scope(context.text, scanned, call.start), plain_select: plain_select(context.text, scanned, call.start), arguments: argument_checks(&definition, &bound, &rendered), call: None, - output, }; - if !check.arguments.is_empty() || check.output.is_some() { + if !check.arguments.is_empty() { host.record_check(check, (call.start, call.end)); } } @@ -662,11 +650,6 @@ fn call_scope(text: &str, scanned: &[Token], offset: usize) -> CallScope { } } -/// The branch of its scope's set operation the call at byte `offset` of `text` stands in. -fn call_branch(text: &str, scanned: &[Token], offset: usize) -> Option { - top_level_parts(text, scanned)?.branch_of(text, scanned, offset) -} - /// Whether the call at byte `offset` of `text` stands in its scope's one select. fn plain_select(text: &str, scanned: &[Token], offset: usize) -> bool { top_level_parts(text, scanned).is_some_and(|parts| parts.plain_select(text, scanned, offset)) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs index 9991907baf..209ddd5283 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs @@ -2,7 +2,7 @@ use crate::sql_macros::_helpers::header::unquoted; use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token}; -use crate::sql_macros::constants::{AS_KEYWORD, AT_SIGN, COMMA, SELECT_KEYWORD, WITH_KEYWORD}; +use crate::sql_macros::constants::{AT_SIGN, COMMA, SELECT_KEYWORD, WITH_KEYWORD}; use crate::sql_macros::models::{CallPosition, CallScope}; /// Tokens before the `(` of `DISTINCT ON (...)`. @@ -11,10 +11,6 @@ const DISTINCT_ON_TOKENS: usize = 2; const QUERY_OPENERS: [&str; 4] = ["SELECT", "WITH", "VALUES", "FROM"]; /// Keywords that combine the selects of a query. const SET_OPERATIONS: [&str; 3] = ["UNION", "INTERSECT", "EXCEPT"]; -/// A star, which an `EXCEPT` column list may follow as a modifier, not a set operation. -const STAR: &str = "*"; -/// The words that may stand between a set operation and its parenthesised right branch. -const SET_OPERATION_MODIFIERS: [&str; 3] = ["ALL", "DISTINCT", "NAME"]; /// Tokens from `@` to a macro call's name: `@`, the kind, `.` and the name. const CALL_PREFIX_TOKENS: usize = 4; /// Keywords that open the clause a call stands in, read backwards from the call. @@ -256,48 +252,6 @@ impl<'a> CallSite<'a> { } } - /// The alias of the select-list item the call is the whole of, as `@sql.m(x) AS name`. - pub(crate) fn whole_item_alias(&self) -> Option { - let opens_item: bool = self.previous_token().is_some_and(|token| { - matches!( - token.text(self.sql).to_ascii_uppercase().as_str(), - "SELECT" | "DISTINCT" | "ALL" | "," - ) - }); - if !opens_item || self.position() != CallPosition::SelectList { - return None; - } - let next: usize = self.next?; - let alias_index: usize = if self.tokens[next].is_word(self.sql, AS_KEYWORD) { - next + 1 - } else { - next - }; - let alias: &Token = self.tokens.get(alias_index)?; - let alias_text: String = alias.text(self.sql).to_ascii_uppercase(); - let named: bool = match alias.kind { - TokenKind::Quoted => true, - TokenKind::Word => { - !SELECT_LIST_ENDS.contains(&alias_text.as_str()) - && !EXPRESSION_OPERATOR_WORDS.contains(&alias_text.as_str()) - && alias_text != AS_KEYWORD - } - _ => false, - }; - if !named { - return None; - } - let ends_item: bool = match self.tokens.get(alias_index + 1) { - None => true, - Some(after) => { - let text: String = after.text(self.sql).to_ascii_uppercase(); - matches!(text.as_str(), "," | ")" | ";") - || (after.kind == TokenKind::Word && SELECT_LIST_ENDS.contains(&text.as_str())) - } - }; - ends_item.then(|| unquoted(alias.text(self.sql))) - } - /// Whether the token after the call ends a select-list item. fn ends_select_item(&self) -> bool { let Some(next) = self.next_token() else { @@ -615,74 +569,6 @@ impl QueryParts { } } -impl QueryParts { - /// The set-operation branch byte `offset` stands in; `None` where it cannot be counted. - pub(crate) fn branch_of(&self, text: &str, scanned: &[Token], offset: usize) -> Option { - let range: (usize, usize) = match self.scope_of(offset) { - CallScope::WithList => return None, - CallScope::Cte(name) => self.ctes.iter().find(|cte| cte.name == name)?.body, - CallScope::Main => (self.main, text.len()), - }; - let inside: Vec<(usize, &Token)> = scanned - .iter() - .enumerate() - .filter(|(_, token)| range.0 <= token.start && token.end <= range.1) - .collect(); - if inside - .first() - .is_some_and(|(_, token)| token.is_word(text, WITH_KEYWORD)) - { - return None; - } - let mut depth: usize = 0; - let mut branch: usize = 0; - let mut at_call: Option = None; - let mut previous_at_top: Option<&Token> = None; - for (index, token) in inside { - if at_call.is_none() && token.start >= offset { - if depth != 0 { - return None; - } - at_call = Some(branch); - } - if token.kind == TokenKind::Punct { - match token.text(text) { - "(" | "[" | "{" => { - let query: bool = scanned - .get(index + 1) - .is_some_and(|next| opens_query(text, next)); - let branch_start: bool = previous_at_top.is_none_or(|previous| { - let word: &str = previous.text(text); - is_set_operation(word) - || SET_OPERATION_MODIFIERS - .iter() - .any(|modifier| word.eq_ignore_ascii_case(modifier)) - }); - if depth == 0 && query && branch_start { - return None; - } - depth += 1; - } - ")" | "]" | "}" => depth = depth.saturating_sub(1), - _ => {} - } - } - if depth == 0 { - let star_modifier: bool = - previous_at_top.is_some_and(|previous| previous.text(text) == STAR); - if token.kind == TokenKind::Word - && is_set_operation(token.text(text)) - && !star_modifier - { - branch += 1; - } - previous_at_top = Some(token); - } - } - at_call - } -} - /// Whether `token` opens a query, as the first token inside a parenthesis. fn opens_query(text: &str, token: &Token) -> bool { QUERY_OPENERS diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 646741e2ad..731dda31cc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -64,8 +64,6 @@ pub(crate) const CTE_NOUN: &str = "CTE"; /// Clause keywords a call position reads. pub(crate) const WITH_KEYWORD: &str = "WITH"; pub(crate) const SELECT_KEYWORD: &str = "SELECT"; -/// The keyword before an alias. -pub(crate) const AS_KEYWORD: &str = "AS"; /// Punctuation and operators the scans compare against. pub(crate) const COMMA: &str = ","; pub(crate) const AT_SIGN: &str = "@"; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index 08d58a97b6..5d7f5d30b1 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -10,9 +10,6 @@ pub mod spliced_typed_calls; pub mod sql_macro_body_form; pub mod sql_macro_calls; pub mod sql_macro_cycle; -pub mod sql_macro_declared_outputs; -pub mod sql_macro_declared_type; -pub mod sql_macro_output_names; pub mod sql_macro_parameter_spans; pub mod typed_analysis_pass; pub mod typed_analysis_sql; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs deleted file mode 100644 index f9c379d720..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_outputs.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::sql_macros::models::{CallScope, DeclaredOutput, SqlMacroCheck}; - -/// The select items, in any set-operation branch, of a model's top-level queries that are one call. -#[must_use] -pub fn sql_macro_declared_outputs(checks: &[SqlMacroCheck]) -> Vec { - checks - .iter() - .filter_map(|check| { - let output = check.output.as_ref()?; - let cte: Option = match &check.scope { - CallScope::Main => None, - CallScope::Cte(name) => Some(name.clone()), - CallScope::WithList => return None, - }; - Some(DeclaredOutput { - cte, - name: output.name.clone(), - returns: output.returns.clone(), - branch: output.branch, - }) - }) - .collect() -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs deleted file mode 100644 index 2e7aa03ef5..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_declared_type.rs +++ /dev/null @@ -1,40 +0,0 @@ -use crate::constants::UNKNOWN_SQL_TYPE; -use crate::sql_macros::models::DeclaredOutput; -use crate::type_system::main::normalize_type::normalize_type; -use crate::type_system::models::NormalizedType; - -/// The declared `returns` type an item takes over its inferred type: unknown or identical. -#[must_use] -pub fn sql_macro_declared_type( - declared: &[DeclaredOutput], - (cte, branch, name): (Option<&str>, usize, &str), - inferred: Option<&str>, - dialect: &str, -) -> Option { - let output: &DeclaredOutput = declared.iter().find(|output| { - output.name.eq_ignore_ascii_case(name) - && output.branch == branch - && match (&output.cte, cte) { - (None, None) => true, - (Some(declared_cte), Some(cte)) => declared_cte.eq_ignore_ascii_case(cte), - _ => false, - } - })?; - let Some(inferred) = inferred.filter(|inferred| { - !inferred.is_empty() && !inferred.eq_ignore_ascii_case(UNKNOWN_SQL_TYPE) - }) else { - return Some(output.returns.clone()); - }; - let Some(inferred_type) = normalized(inferred, dialect) else { - return Some(output.returns.clone()); - }; - (normalized(&output.returns, dialect) == Some(inferred_type)).then(|| output.returns.clone()) -} - -/// `data_type` as compared, or `None` where it does not normalise. -fn normalized(data_type: &str, dialect: &str) -> Option { - match normalize_type(data_type, dialect) { - Ok(normalization) => Some(normalization.normalized), - Err(_unnormalizable) => None, - } -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs deleted file mode 100644 index 5e06e4bbf5..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_output_names.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::sql_macros::_helpers::expansion::sql_macro_calls; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::placement::{CallSite, QueryParts, top_level_parts}; -use crate::sql_macros::models::{CallScope, SqlMacroCall}; - -/// The `(top-level CTE, branch, alias)` of each select item of `text` that is one SQL macro call. -#[must_use] -pub fn sql_macro_output_names(text: &str) -> Vec<(Option, usize, String)> { - let Ok(calls) = sql_macro_calls(text) else { - return Vec::new(); - }; - if calls.is_empty() { - return Vec::new(); - } - let Ok(scanned) = tokens(text) else { - return Vec::new(); - }; - let Some(parts) = top_level_parts(text, &scanned) else { - return Vec::new(); - }; - calls - .iter() - .filter(|call| !call.in_python) - .filter_map(|call| whole_item(text, (&scanned, &parts), call)) - .collect() -} - -/// The `(top-level CTE, branch, alias)` of `call` when it is a whole aliased select item. -fn whole_item( - text: &str, - (scanned, parts): (&[Token], &QueryParts), - call: &SqlMacroCall, -) -> Option<(Option, usize, String)> { - let cte: Option = match parts.scope_of(call.start) { - CallScope::Main => None, - CallScope::Cte(name) => Some(name), - CallScope::WithList => return None, - }; - let branch: usize = parts.branch_of(text, scanned, call.start)?; - CallSite::new(text, scanned, (call.start, call.end), call.replaced_end) - .whole_item_alias() - .map(|alias| (cte, branch, alias)) -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index c4044a2b72..c2912df89f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -293,31 +293,6 @@ pub struct SqlMacroCheck { pub arguments: Vec, /// The call's authored file location and where its text ends, when the host knows them. pub call: Option<(SqlMacroLocation, (usize, usize))>, - /// The output column an expression call is the whole select-list item of, and its type. - pub output: Option, -} - -/// A select item that is one expression macro call, typed by the macro's `returns`. -#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct DeclaredOutput { - /// The top-level CTE the item is in, or `None` for the main query. - pub cte: Option, - /// The item's alias, unquoted. - pub name: String, - pub returns: String, - /// The select it stands in: 0, or its branch of a set operation, left to right. - pub branch: usize, -} - -/// A select-list item that is one expression macro call under an alias, as `@sql.m(x) AS name`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TypedOutput { - /// The alias, unquoted. - pub name: String, - /// The macro's declared `returns` type. - pub returns: String, - /// The select the item stands in: 0, or its branch of a set operation, left to right. - pub branch: usize, } /// The top-level part of the caller's query a call stands in. diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index 3c25706d1a..31340ce436 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -4,8 +4,6 @@ mod helpers; mod test_definitions; #[path = "tests/test_expansion.rs"] mod test_expansion; -#[path = "tests/test_output_types.rs"] -mod test_output_types; #[path = "tests/test_typed_calls.rs"] mod test_typed_calls; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index 95cb1c43d2..0205c590c0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -5,8 +5,7 @@ use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; use crate::sql_macros::models::{ - CallScope, ExpandedText, RenderedDeclarations, SqlMacroCheck, SqlMacroDefinition, TextContext, - TextSplice, TypedOutput, + ExpandedText, RenderedDeclarations, SqlMacroDefinition, TextContext, TextSplice, }; use crate::sql_macros::tests::test_types::{AsciiArguments, FakeHost}; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -155,19 +154,3 @@ impl sqlbuild_render::macro_arguments::types::ArgumentHost for AsciiArguments { None } } - -/// A check recording output `label` of `shout()`, typed VARCHAR, at `scope`'s `branch`. -pub(super) fn typed_check(scope: CallScope, branch: usize) -> SqlMacroCheck { - SqlMacroCheck { - macro_name: "shout".to_owned(), - scope, - plain_select: true, - arguments: Vec::new(), - call: None, - output: Some(TypedOutput { - name: "label".to_owned(), - returns: "VARCHAR".to_owned(), - branch, - }), - } -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs deleted file mode 100644 index 9861f19e11..0000000000 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_output_types.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::sql_macros::main::sql_macro_declared_outputs::sql_macro_declared_outputs; -use crate::sql_macros::main::sql_macro_declared_type::sql_macro_declared_type; -use crate::sql_macros::main::sql_macro_output_names::sql_macro_output_names; -use crate::sql_macros::models::CallScope; -use crate::sql_macros::tests::helpers::typed_check; -use crate::sql_macros::tests::test_types::{OutputNamesTestCase, OutputTypeTestCase}; - -#[test] -fn given_a_typed_macro_output_when_typing_the_column_then_returns_fills_unknown_and_refines() { - let test_cases = [ - OutputTypeTestCase { - description: "unknown inferred type takes returns", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", None), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "an identical type takes returns' spelling", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "LABEL", Some("TEXT")), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a different known family keeps the inferred type", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("BIGINT")), - expected_type: None, - }, - OutputTypeTestCase { - description: "a longer text type stays", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("VARCHAR(100)")), - expected_type: None, - }, - OutputTypeTestCase { - description: "the unknown placeholder takes returns", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("UNKNOWN")), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a known time type is not unknown", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("TIME")), - expected_type: None, - }, - OutputTypeTestCase { - description: "a known array type is not unknown", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "label", Some("TEXT[]")), - expected_type: None, - }, - OutputTypeTestCase { - description: "another column is untouched", - check: typed_check(CallScope::Main, 0), - item: (None, 0, "other", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in a CTE types that CTE's item", - check: typed_check(CallScope::Cte("base".to_owned()), 0), - item: (Some("BASE"), 0, "label", None), - expected_type: Some("VARCHAR"), - }, - OutputTypeTestCase { - description: "a call in a CTE leaves a main-query item of that name", - check: typed_check(CallScope::Cte("base".to_owned()), 0), - item: (None, 0, "label", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in another branch of a set operation leaves this one", - check: typed_check(CallScope::Main, 1), - item: (None, 0, "label", None), - expected_type: None, - }, - OutputTypeTestCase { - description: "a call in a later branch of a set operation types that branch", - check: typed_check(CallScope::Cte("base".to_owned()), 1), - item: (Some("base"), 1, "label", Some("TEXT")), - expected_type: Some("VARCHAR"), - }, - ]; - - for test_case in test_cases { - let (cte, branch, name, inferred) = test_case.item; - let declared = sql_macro_declared_outputs(&[test_case.check]); - let typed = sql_macro_declared_type(&declared, (cte, branch, name), inferred, "duckdb"); - - assert_eq!( - typed.as_deref(), - test_case.expected_type, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_model_sql_when_reading_sql_macro_outputs_then_only_whole_aliased_items_count() { - let test_cases = [ - OutputNamesTestCase { - description: "aliased whole items, with and without AS", - sql: "SELECT @sql.a(x) AS one, @sql.b(y) two, @sql.c(z) AS \"Three\" FROM t", - expected_names: &[(None, 0, "one"), (None, 0, "two"), (None, 0, "Three")], - }, - OutputNamesTestCase { - description: "calls inside expressions or without an alias", - sql: "SELECT @sql.a(x) || 'x' AS one, UPPER(@sql.b(y)) AS two, @sql.c(z) FROM t", - expected_names: &[], - }, - OutputNamesTestCase { - description: "a call in a CTE, and ones in each branch of a set operation", - sql: "WITH c AS (SELECT @sql.a(x) AS one FROM t) SELECT @sql.b(y) AS two FROM c UNION ALL SELECT 'x' FROM c UNION ALL SELECT @sql.c(z) AS three FROM c", - expected_names: &[(Some("c"), 0, "one"), (None, 0, "two"), (None, 2, "three")], - }, - OutputNamesTestCase { - description: "a star EXCEPT modifier is not a set operation", - sql: "SELECT * EXCEPT (a) FROM t UNION ALL SELECT @sql.b(y) AS two FROM t", - expected_names: &[(None, 1, "two")], - }, - OutputNamesTestCase { - description: "calls in subqueries, a nested WITH or beside a parenthesised branch", - sql: "WITH c AS (WITH d AS (SELECT 1 AS x) SELECT @sql.a(x) AS one FROM d), e AS (SELECT * FROM (SELECT @sql.b(y) AS two FROM t) AS s) SELECT @sql.c(z) AS three FROM c UNION ALL (SELECT 'x' FROM e)", - expected_names: &[], - }, - OutputNamesTestCase { - description: "a call in the main query after CTEs", - sql: "WITH c AS (SELECT x FROM t) SELECT x, @sql.a(x) AS one FROM c", - expected_names: &[(None, 0, "one")], - }, - ]; - - for test_case in test_cases { - let names: Vec<(Option, usize, String)> = sql_macro_output_names(test_case.sql); - let found: Vec<(Option<&str>, usize, &str)> = names - .iter() - .map(|(cte, branch, name)| (cte.as_deref(), *branch, name.as_str())) - .collect(); - - assert_eq!(found, test_case.expected_names, "{}", test_case.description); - } -} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 6451c6483c..b655b96bbe 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -20,6 +20,10 @@ const SAME: (&str, &str) = ( "same", "MACRO (description \"Same\", arguments (value VARCHAR), returns VARCHAR);\n\n@value\n", ); +const NOTHING: (&str, &str) = ( + "nothing", + "MACRO (description \"Nothing\", arguments (value VARCHAR), returns BIGINT);\n\nNULL\n", +); const PLACES: (&str, &str) = ( "places", "MACRO (description \"Places\", arguments (value BIGINT), returns BIGINT);\n\n\ @@ -89,6 +93,13 @@ fn given_expression_macro_calls_when_expanding_then_each_expansion_is_recorded_w @sql.shout(@enum(\"order_status\").OPEN) AS s FROM orders AS o", expected_calls: &["places BIGINT: o.a + 2", "shout VARCHAR: UPPER('open')"], }, + TypedCallTestCase { + description: "a call in a later set-operation branch", + macros: &[NOTHING], + sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"stg_orders\") AS o\nUNION ALL\n\ + SELECT @sql.nothing(o.status) AS v FROM __ref(\"stg_orders\") AS o", + expected_calls: &["nothing BIGINT: NULL"], + }, TypedCallTestCase { description: "a call inside a Python macro argument is not recorded", macros: &[SHOUT], diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index de2904e274..06bf27cb5c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -63,27 +63,6 @@ pub(super) struct PythonArgumentTestCase { /// Argument parsing without Unicode name lookups, which these arguments never use. pub(super) struct AsciiArguments; -/// One recorded call, one analysed select item and the type the item then has, if changed. -pub(super) struct OutputTypeTestCase { - pub(super) description: &'static str, - pub(super) check: crate::sql_macros::models::SqlMacroCheck, - /// The item's CTE (`None` for the main query), set-operation branch, name and inferred type. - pub(super) item: ( - Option<&'static str>, - usize, - &'static str, - Option<&'static str>, - ), - pub(super) expected_type: Option<&'static str>, -} - -/// Model SQL and the scope and alias of each whole-item SQL macro call. -pub(super) struct OutputNamesTestCase { - pub(super) description: &'static str, - pub(super) sql: &'static str, - pub(super) expected_names: &'static [(Option<&'static str>, usize, &'static str)], -} - pub(super) struct MacroDescriptionTestCase { pub(super) description: &'static str, pub(super) source: &'static str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 5ef5b8dec2..2a7f6664d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -586,6 +586,7 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult( enum_columns, enum_declarations: declarations(enums)?, constant_declarations: declarations(constants)?, + typed_calls: finished.typed_calls, }); Ok((record, facts)) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 9fedfc925e..afe6dbf8d4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -50,6 +50,7 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result .iter() .map(declaration) .collect(), + typed_calls: facts.typed_calls.clone(), }) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index 15c69a0ea5..f0b485612c 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -132,6 +132,7 @@ impl RowWriter { relative_path, query_sql: _, authored_sql: _, + typed_calls: _, config, authored_config_keys, logical_schema, @@ -348,6 +349,7 @@ impl RowReader<'_> { relative_path: self.text()?, query_sql: String::new(), authored_sql: String::new(), + typed_calls: Vec::new(), config: { let count: usize = self.count()?; let mut config: BTreeMap = BTreeMap::new(); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index eda014fbd7..4a94856b59 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -94,6 +94,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { enum_columns: vec!["status".to_owned()], enum_declarations: vec![declaration.clone()], constant_declarations: vec![declaration], + typed_calls: Vec::new(), declared_audit_count: 3, targeting_test_count: 4, empty_input_only_test_count: 0, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index d8a6b21248..f045e25276 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use sqlbuild_analysis::sql_macros::models::TypedCall; use crate::errors::RowsError; use serde_json::Value; @@ -569,6 +570,9 @@ pub struct Model { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + #[serde(default)] + pub typed_calls: Vec, pub declared_audit_count: u32, pub targeting_test_count: u32, #[serde(skip_deserializing)] @@ -877,6 +881,8 @@ pub struct ModelRow { pub enum_columns: Vec, pub enum_declarations: Vec, pub constant_declarations: Vec, + /// Each expression SQL macro call in `query_sql`, by byte range. + pub typed_calls: Vec, } /// A model's declared schema entry. diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index fe32b68f45..5af9afca09 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -322,6 +322,15 @@ impl Builder<'_> { direct_projection: column_name(value).is_some(), direct_column: column_name(unwrapped), cast_type, + span: range + .as_ref() + .filter(|range| !range.is_empty()) + .map(|range| { + ( + self.tokens.start(range.start), + self.tokens.end(range.end - 1), + ) + }), start: range.and_then(|range| self.tokens.leaf_start(&range)), }) } diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index 8f77a5c64b..db8a95d7d2 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -89,6 +89,8 @@ pub(crate) struct ExpressionItem { pub(crate) direct_column: Option, pub(crate) cast_type: Option, pub(crate) start: Option, + /// The normalized characters from the expression's first token to its last, alias excluded. + pub(crate) span: Option<(usize, usize)>, } #[derive(Debug)] @@ -251,6 +253,14 @@ impl<'a> ModelSource<'a> { .unwrap_or_else(|| self.bytes.len() - 1) } + /// The model's byte range from normalized character `start` to the character before `end`. + pub(crate) fn byte_range(&self, start: usize, end: usize) -> (usize, usize) { + let last: usize = self.bytes.len() - 1; + let first: usize = self.bytes[self.original(start).min(last)]; + let after: usize = self.bytes[(self.original(end.saturating_sub(1)) + 1).min(last)]; + (first, after.max(first)) + } + /// The 1-based line and column of normalized character `offset`. pub(crate) fn line_column(&self, offset: usize) -> (u64, u64) { let character = self.original(offset); diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs index d760285923..7765ad08c9 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs @@ -1,10 +1,8 @@ -use std::cell::OnceCell; - -use sqlbuild_analysis::sql_macros::main::sql_macro_output_names::sql_macro_output_names; +use sqlbuild_analysis::sql_macros::models::TypedCall; use crate::constants::ENFORCED_CONTRACT; use crate::models::{Column, Fault, Model, RuleMetadata}; -use crate::query_facts::models::{ItemFacts, QueryFacts, SelectFacts, SetFacts}; +use crate::query_facts::models::{ExpressionItem, ItemFacts, QueryFacts, SelectFacts, SetFacts}; use crate::rules::_helpers::evaluation::{ ParsedModel, plain_select, reads_dependency, sole_table_name, }; @@ -15,10 +13,6 @@ struct OutputScope<'a> { columns: &'a [Column], set_branch: bool, by_name: bool, - /// Where SQL macro call items are read: the main query (`Some(None)`), a CTE, or nowhere. - macro_scope: Option>, - /// The first select's branch of the scope's set operation, left to right. - branch: usize, } struct OutputRule<'a> { @@ -26,8 +20,6 @@ struct OutputRule<'a> { model: &'a Model, rule: &'a RuleMetadata, faults: &'a FaultCollector, - /// The main query's outputs that are one SQL macro call each, read when first needed. - sql_macro_outputs: OnceCell, usize, String)>>, } pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { @@ -54,18 +46,14 @@ pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &F model, rule, faults, - sql_macro_outputs: OnceCell::new(), }; if let Some((boundary, columns)) = final_cte_output_boundary(query, &model.columns) { - let final_cte: Option<&str> = query.ctes.last().map(|cte| cte.name.as_str()); evaluator.evaluate_set( &boundary.body, OutputScope { columns: &columns, set_branch: false, by_name: false, - macro_scope: final_cte.map(Some), - branch: 0, }, ); return; @@ -77,8 +65,6 @@ pub(crate) fn evaluate(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &F columns: &columns, set_branch: false, by_name: false, - macro_scope: Some(None), - branch: 0, }, ); } @@ -142,17 +128,9 @@ impl OutputRule<'_> { columns: scope.columns, set_branch: true, by_name: *by_name, - macro_scope: scope.macro_scope, - branch: scope.branch, }; self.evaluate_set(left, branch_scope); - self.evaluate_set( - right, - OutputScope { - branch: scope.branch + set_branches(left), - ..branch_scope - }, - ); + self.evaluate_set(right, branch_scope); } SetFacts::Other => self.push( None, @@ -213,7 +191,9 @@ impl OutputRule<'_> { || column.type_proven && direct && !scope.set_branch || column.type_proven && !explicit_cast - && self.sql_macro_output(scope, output_name.as_deref()); + && self + .whole_call(expression) + .is_some_and(|call| call.returns.is_some()); if passes { continue; } @@ -241,23 +221,16 @@ impl OutputRule<'_> { } } - /// Whether output `name` in `scope`'s select is one SQL macro call, typed by its `returns`. - fn sql_macro_output(&self, scope: OutputScope<'_>, name: Option<&str>) -> bool { - let (Some(macro_scope), Some(name)) = (scope.macro_scope, name) else { - return false; - }; - self.sql_macro_outputs - .get_or_init(|| sql_macro_output_names(&self.model.authored_sql)) - .iter() - .any(|(cte, branch, output)| { - output.eq_ignore_ascii_case(name) - && *branch == scope.branch - && match (cte.as_deref(), macro_scope) { - (None, None) => true, - (Some(cte), Some(scope)) => cte.eq_ignore_ascii_case(scope), - _ => false, - } - }) + /// The expression SQL macro call `expression` is the whole of, found by its byte range. + fn whole_call(&self, expression: &ExpressionItem) -> Option<&TypedCall> { + let (start, end) = expression.span?; + let (start, end) = self.parsed.source.byte_range(start, end); + let sql: &str = &self.model.query_sql; + self.model.typed_calls.iter().rev().find(|call| { + call.start == start + && end <= call.end + && sql.get(end..call.end).is_some_and(only_trivia) + }) } fn push(&self, offset: Option, message: String, remediation: String) { @@ -274,11 +247,23 @@ impl OutputRule<'_> { } } -/// The selects of `set_expr`'s set operation, a nested query counting as one. -fn set_branches(set_expr: &SetFacts) -> usize { - match set_expr { - SetFacts::SetOperation { left, right, .. } => set_branches(left) + set_branches(right), - SetFacts::Select(_) | SetFacts::Query(_) | SetFacts::Other => 1, +/// Whether `text` holds only whitespace and comments. +fn only_trivia(mut text: &str) -> bool { + loop { + text = text.trim_start(); + if text.is_empty() { + return true; + } + if let Some(comment) = text.strip_prefix("--") { + text = comment.split_once('\n').map_or("", |(_, rest)| rest); + } else if let Some(comment) = text.strip_prefix("/*") { + let Some((_, rest)) = comment.split_once("*/") else { + return false; + }; + text = rest; + } else { + return false; + } } } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py index 9620ef8e76..d5bfa88f92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py @@ -202,7 +202,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( unexpected_fragments=_UNPROVEN, ), SqlMacroCompileTestCase( - description="a_call_inside_a_larger_expression_is_typed_from_its_sql", + description="a_call_inside_a_larger_expression_is_typed_but_needs_a_cast_for_the_rule", files={ **_TYPED_RULES_AND_MACROS, **contract_model( @@ -216,11 +216,10 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( }, expected_returncode=1, expected_fragments=( - "warning[K003]: column 'status_bar' type could not be proven against declared " - "VARCHAR", 'error[SQBRCONTRACT105]: output "status_bar" is calculated without an outer ' "explicit cast", ), + unexpected_fragments=("K003",), ), SqlMacroCompileTestCase( description="a_time_body_keeps_its_inferred_type", From 531a0c0c014ca07f2ee11a46f2e456478447d60d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:26:26 +0100 Subject: [PATCH 05/28] feat: make SQL macro returns optional and check it against the body An expression macro may omit returns; its calls are then typed by inference alone. When returns is declared, the body is typed once from stand-ins for its declared arguments, and a known type of another family is reported as P024 at the macro file. K003 and the explicit-cast rule suggest declaring returns when an output is one call to a macro without it. --- .../src/contracts/_helpers/columns.rs | 15 +- .../sqlbuild-analysis/src/contracts/models.rs | 2 + .../src/contracts/tests/helpers.rs | 1 + .../src/query_analysis/main/output_types.rs | 34 +++- .../src/sql_macros/_helpers/arguments.rs | 7 +- .../src/sql_macros/_helpers/definition.rs | 8 +- .../src/sql_macros/_helpers/expansion.rs | 6 + .../src/sql_macros/_helpers/mod.rs | 1 + .../src/sql_macros/_helpers/returns_check.rs | 167 ++++++++++++++++++ .../src/sql_macros/constants.rs | 4 + .../src/sql_macros/main/mod.rs | 1 + .../sql_macros/main/untyped_call_outputs.rs | 53 ++++++ .../src/sql_macros/tests/test_definitions.rs | 6 - .../src/sql_macros/tests/test_expansion.rs | 56 +++++- .../src/sql_macros/tests/test_typed_calls.rs | 30 ++++ .../_helpers/contracts/model_contracts.rs | 2 + .../_helpers/render/sql_macro_host.rs | 4 +- .../rules/_helpers/explicit_output_types.rs | 23 ++- .../_helpers/corpus/render_failure_cases.py | 11 ++ scripts/compiler_differential/constants.py | 1 + .../main/compile/sql_macros/helpers.py | 13 +- .../sql_macros/test_sql_macro_build.py | 7 +- .../sql_macros/test_sql_macro_errors.py | 16 +- .../sql_macros/test_sql_macro_returns.py | 142 +++++++++++++++ .../sql_macros/test_sql_macro_types.py | 20 ++- 25 files changed, 575 insertions(+), 55 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index eacc966c75..8603bf2d2d 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -109,6 +109,7 @@ pub(crate) fn model_contract_diagnostics( contract_enforced, unchecked: &unchecked, types, + untyped_calls: &model.untyped_call_outputs, }; diagnostics.extend(column_type(&checked, &check)?); } @@ -120,6 +121,8 @@ struct TypeCheck<'a> { contract_enforced: bool, unchecked: &'a HashSet<&'a str>, types: &'a TypeComparer<'a>, + /// Output columns that are one call to a SQL macro without `returns`, and the macro. + untyped_calls: &'a [(String, String)], } struct TypedColumn<'a> { @@ -269,7 +272,17 @@ fn column_type( column_name: name.to_owned(), message: UNPROVEN_TYPE_OUTPUT_MESSAGE.to_owned(), }), - help: UNKNOWN_TYPE_HELP.to_owned(), + help: match check + .untyped_calls + .iter() + .find(|(output, _)| output.eq_ignore_ascii_case(name)) + { + Some((_, macro_name)) => format!( + "declare returns {declared_type} in SQL macro {macro_name}(), or add an \ + explicit CAST if this declared type should be checked statically" + ), + None => UNKNOWN_TYPE_HELP.to_owned(), + }, })); }; if check.types.types_equal(declared_type, inferred_type)? == TypeComparison::Equal { diff --git a/crates/sqlbuild-analysis/src/contracts/models.rs b/crates/sqlbuild-analysis/src/contracts/models.rs index 656f80975a..868b8904eb 100644 --- a/crates/sqlbuild-analysis/src/contracts/models.rs +++ b/crates/sqlbuild-analysis/src/contracts/models.rs @@ -23,6 +23,8 @@ pub struct ContractModel { pub fast_lineage_has_star: bool, pub dynamic_proof: Option, pub unchecked_output_columns: Vec, + /// Output columns that are one call to a SQL macro without `returns`, with the macro's name. + pub untyped_call_outputs: Vec<(String, String)>, } /// The declared shape of one model. diff --git a/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs b/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs index e573e13ec7..bb4ff5d21a 100644 --- a/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/contracts/tests/helpers.rs @@ -12,6 +12,7 @@ pub(crate) fn model(contract: Option<&str>, schema: Option) -> C fast_lineage_has_star: false, dynamic_proof: None, unchecked_output_columns: Vec::new(), + untyped_call_outputs: Vec::new(), } } diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs index c7d83d7d89..911d42b271 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; -use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_complete_outputs}; +use crate::query_analysis::compatibility_types::{ + NULL_TYPE, infer_complete_outputs, infer_outputs, +}; /// The output columns and inferred types of the one query in `sql`; `None` if a source is untyped. #[must_use] @@ -32,6 +34,36 @@ pub fn output_types( Some(outputs) } +/// The output columns and types of the one query in `sql`, typed by full type annotation. +#[must_use] +pub fn annotated_output_types( + sql: &str, + dialect: &str, + schema: &ValidationSchema, +) -> Option)>> { + let dialect: DialectType = dialect.parse().ok()?; + let mut statements: Vec = + polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()).ok()?; + if statements.len() != 1 { + return None; + } + let mut expression: Expression = statements.pop()?; + let mapping = polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( + schema, dialect, + ); + polyglot_sql::optimizer::annotate_types::annotate_types( + &mut expression, + Some(&mapping as &dyn polyglot_sql::schema::Schema), + Some(dialect), + ); + Some( + infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) + .into_iter() + .map(|(name, data_type)| (name, data_type.filter(|value| !is_null_type(value)))) + .collect(), + ) +} + /// Whether `value` is the placeholder inference uses for a `NULL` literal's type. fn is_null_type(value: &str) -> bool { value == NULL_TYPE diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs index 76769a2286..8a4d955e15 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs @@ -2,6 +2,7 @@ use crate::sql_macros::_helpers::header::{split_top_level, unquoted}; use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::returns_check::adapter_dialect; use crate::sql_macros::constants::{ ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, }; @@ -493,11 +494,7 @@ pub(crate) fn is_atomic(sql: &str) -> bool { /// The family of a declared type, normalized as schema declaration types are for the adapter. pub(crate) fn type_family(data_type: &str, adapter: &str) -> TypeFamily { use crate::type_system::models::TypeFamily as Normalized; - let dialect: &str = match adapter { - "motherduck" => "duckdb", - "" => "generic", - other => other, - }; + let dialect: &str = adapter_dialect(adapter); let normalized = normalize_type(data_type, dialect) .or_else(|_| normalize_type(data_type, "generic")) .map(|normalization| normalization.normalized.family); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs index 0631d5a5af..099a9940d6 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs @@ -154,19 +154,13 @@ fn body_calls(name: &str, body: &str, in_body: Locate<'_>) -> Result Ok(calls) } -/// Rejects an expression macro without `returns` and another form with it. +/// Rejects `returns` on a CTE-list or column-list macro. fn check_returns( name: &str, (form, returns): (SqlMacroForm, Option<&String>), location: SqlMacroLocation, ) -> Result<(), SqlMacroError> { match (form, returns) { - (SqlMacroForm::Expression, None) => Err(SqlMacroError::new( - INVALID_DEFINITION_CODE, - format!("expression macro {name}() needs returns in its MACRO header"), - ) - .at(location) - .helped("declare the type the expression returns, such as returns DECIMAL(18, 2)")), (SqlMacroForm::CteList | SqlMacroForm::ColumnList, Some(_)) => Err(SqlMacroError::new( INVALID_DEFINITION_CODE, format!( diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index e76d4ac5d5..c452652b41 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -15,6 +15,7 @@ use crate::sql_macros::_helpers::literal_positions::{ use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; use crate::sql_macros::_helpers::placement::top_level_parts; use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::returns_check::check_body_returns; use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ ARGUMENT_CODE, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, MISPLACED_CALL_CODE, @@ -217,6 +218,11 @@ pub(crate) fn expand_call( }, ) .map_err(|failure| located(failure, &body_location(&definition)))?; + check_body_returns( + &definition, + &body, + (host.adapter(), body_location(&definition)), + )?; let expansion: BodyExpansion = expand_body(&definition, &body, &rendered) .map_err(|error| error.at(body_location(&definition)))?; if call.in_python { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs index 90abead517..3296e7685d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs @@ -8,4 +8,5 @@ pub(crate) mod literal_positions; pub(crate) mod parameters; pub(crate) mod placement; pub(crate) mod probes; +pub(crate) mod returns_check; pub(crate) mod typed_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs new file mode 100644 index 0000000000..8069d96f4d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs @@ -0,0 +1,167 @@ +//! An expression macro's body typed from its declared arguments and checked against `returns`. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use polyglot_sql::ValidationSchema; +use serde_json::{Value, json}; + +use crate::contracts::_helpers::type_comparison::{TypeComparer, TypeComparison}; +use crate::query_analysis::main::output_types::annotated_output_types; +use crate::sql_macros::_helpers::arguments::type_family; +use crate::sql_macros::_helpers::body::expand_body; +use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::sql_macros::constants::{ + INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, +}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + ArgumentShape, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, SqlMacroForm, + SqlMacroLocation, TypeFamily, +}; + +/// Each body check's verdict, by the probe it ran, so a macro is typed once per process. +static VERDICTS: OnceLock>>> = OnceLock::new(); + +/// The dialect an adapter's SQL is analysed in. +pub(crate) fn adapter_dialect(adapter: &str) -> &str { + match adapter { + "motherduck" => "duckdb", + "" => "generic", + other => other, + } +} + +/// The type a parameter's stand-in column has, or `None` for an untyped stand-in. +fn stand_in_type(kind: &ParameterKind) -> Option<&str> { + match kind { + ParameterKind::Expression { data_type } | ParameterKind::Constant { data_type } => { + Some(data_type) + } + ParameterKind::Collection { data_type, .. } => Some(data_type), + ParameterKind::Any + | ParameterKind::Enum { .. } + | ParameterKind::Relation { .. } + | ParameterKind::ColumnOf { .. } => None, + } +} + +/// The name of parameter `index`'s stand-in column. +fn stand_in_column(index: usize) -> String { + format!("p{index}") +} + +/// The body's type with typed stand-ins for its arguments, unless it reads relations or columns. +fn body_probe(definition: &SqlMacroDefinition, body: &ExpandedText) -> Option<(String, Value)> { + let mut arguments: Vec = Vec::with_capacity(definition.parameters.len()); + let mut columns: Vec = Vec::with_capacity(definition.parameters.len()); + for (index, parameter) in definition.parameters.iter().enumerate() { + if matches!( + parameter.kind, + ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. } + ) { + return None; + } + let column: String = stand_in_column(index); + let sql: String = match parameter.kind { + ParameterKind::Collection { .. } => format!("({STAND_IN_RELATION}.{column})"), + _ => format!("{STAND_IN_RELATION}.{column}"), + }; + columns.push(json!({ + "name": column, + "type": stand_in_type(¶meter.kind).unwrap_or(UNKNOWN_TYPE_HINT), + })); + arguments.push(RenderedArgument { + parameter: index, + sql, + shape: ArgumentShape::Expression, + calls: Vec::new(), + }); + } + let expansion = expand_body(definition, body, &arguments).ok()?; + let (typed, _) = typed_sql(&expansion.sql, &expansion.calls); + let probe: String = format!("SELECT ({typed}) AS b FROM {STAND_IN_RELATION}"); + let table: Value = json!({"name": STAND_IN_RELATION, "columns": columns}); + Some((probe, table)) +} + +/// The body's inferred type, when inference knows it completely. +fn inferred_body_type(probe: &str, table: Value, dialect: &str) -> Option { + let schema: ValidationSchema = serde_json::from_value(json!({ "tables": [table] })).ok()?; + let outputs = annotated_output_types(probe, dialect, &schema)?; + let (_, data_type) = outputs.into_iter().next()?; + data_type.filter(|inferred| !inferred.to_ascii_uppercase().contains(UNKNOWN_TYPE_HINT)) +} + +/// Whether a body of type `inferred` disagrees with `returns`: in another type family, or, for +/// types outside the scalar families, of another type. Within one family the call's own +/// inference decides, since stand-ins lose what literal arguments tell it. +fn disagrees(returns: &str, inferred: &str, adapter: &str) -> bool { + let (declared, found) = ( + type_family(returns, adapter), + type_family(inferred, adapter), + ); + if declared != TypeFamily::Unknown && found != TypeFamily::Unknown { + return declared != found; + } + matches!( + TypeComparer::new(adapter_dialect(adapter)).types_equal(returns, inferred), + Ok(TypeComparison::Different) + ) +} + +/// Rejects an expression macro whose body's known type is not its declared `returns`. +pub(crate) fn check_body_returns( + definition: &SqlMacroDefinition, + body: &ExpandedText, + (adapter, location): (&str, SqlMacroLocation), +) -> Result<(), SqlMacroError> { + if definition.form != SqlMacroForm::Expression { + return Ok(()); + } + let Some(returns) = definition.returns.as_deref() else { + return Ok(()); + }; + let Some((probe, table)) = body_probe(definition, body) else { + return Ok(()); + }; + let dialect: &str = adapter_dialect(adapter); + let key: String = format!("{dialect}\n{table}\n{probe}"); + let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); + let known: Option> = verdicts + .lock() + .ok() + .and_then(|verdicts| verdicts.get(&key).cloned()); + let inferred: Option = match known { + Some(inferred) => inferred, + None => { + let inferred: Option = inferred_body_type(&probe, table, dialect); + if let Ok(mut verdicts) = verdicts.lock() { + verdicts.insert(key, inferred.clone()); + } + inferred + } + }; + let Some(inferred) = inferred else { + return Ok(()); + }; + let shown: &str = if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) { + VARCHAR_TYPE + } else { + &inferred + }; + if !disagrees(returns, shown, adapter) { + return Ok(()); + } + Err(SqlMacroError::new( + RETURNS_MISMATCH_CODE, + format!( + "macro {}() declares returns {returns}, but its body is {shown}", + definition.name + ), + ) + .at(location) + .helped(format!( + "declare returns {shown}, CAST the body to {returns}, or remove returns to use the inferred type" + ))) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 731dda31cc..7fb6c554c4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -22,6 +22,8 @@ pub const MISPLACED_CALL_CODE: &str = "P021"; pub const PARAMETER_REFERENCE_CODE: &str = "P022"; /// A runtime parameter a SQL macro body uses where SQL requires a literal. pub const LITERAL_REQUIRED_CODE: &str = "P023"; +/// An expression macro whose body's known type is not its declared `returns`. +pub const RETURNS_MISMATCH_CODE: &str = "P024"; /// A declaration a SQL macro body uses that is not visible from the macro file. pub const BODY_SCOPE_CODE: &str = "S026"; @@ -100,6 +102,8 @@ pub(crate) const INFERRED_TEXT_TYPE: &str = "TEXT"; pub(crate) const VARCHAR_TYPE: &str = "VARCHAR"; pub(crate) const FROM_KEYWORD: &str = "FROM"; /// The projection name of an unexpanded star in a probe's analysis. +/// The relation whose columns stand in for a macro's arguments while its body is typed. +pub(crate) const STAND_IN_RELATION: &str = "__sqlbuild_macro_arguments"; pub(crate) const STAR_PROJECTION: &str = "*"; /// The type hint analysis reports for a column whose type it does not know. pub(crate) const UNKNOWN_TYPE_HINT: &str = "UNKNOWN"; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index 5d7f5d30b1..e9bc680a04 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -13,4 +13,5 @@ pub mod sql_macro_cycle; pub mod sql_macro_parameter_spans; pub mod typed_analysis_pass; pub mod typed_analysis_sql; +pub mod untyped_call_outputs; pub mod with_list_sql_macro_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs new file mode 100644 index 0000000000..17595b37af --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs @@ -0,0 +1,53 @@ +use crate::sql_macros::_helpers::header::unquoted; +use crate::sql_macros::_helpers::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::models::TypedCall; + +/// Words that end a select item rather than alias it. +const ITEM_ENDS: [&str; 4] = ["FROM", "WHERE", "UNION", "ORDER"]; + +/// Each named select item of `sql` that is exactly one call to a macro without `returns`, as +/// `(output name, macro name)`. +#[must_use] +pub fn untyped_call_outputs(sql: &str, calls: &[TypedCall]) -> Vec<(String, String)> { + let untyped: Vec<&TypedCall> = calls.iter().filter(|call| call.returns.is_none()).collect(); + if untyped.is_empty() { + return Vec::new(); + } + let Ok(scanned) = tokens(sql) else { + return Vec::new(); + }; + let mut outputs: Vec<(String, String)> = Vec::new(); + for call in untyped { + let Some(first) = scanned.iter().position(|token| token.start == call.start) else { + continue; + }; + let opens_item: bool = first.checked_sub(1).is_some_and(|before| { + let token: &Token = &scanned[before]; + token.is(sql, ",") || token.is_word(sql, "SELECT") || token.is_word(sql, "DISTINCT") + }); + let Some(after) = scanned.iter().position(|token| token.start >= call.end) else { + continue; + }; + if !opens_item { + continue; + } + let alias: usize = if scanned[after].is_word(sql, "AS") { + after + 1 + } else { + after + }; + let Some(name) = scanned.get(alias).filter(|token| { + matches!(token.kind, TokenKind::Word | TokenKind::Quoted) + && !ITEM_ENDS.iter().any(|word| token.is_word(sql, word)) + }) else { + continue; + }; + let closes_item: bool = scanned + .get(alias + 1) + .is_none_or(|next| next.is(sql, ",") || next.kind == TokenKind::Word); + if closes_item { + outputs.push((unquoted(name.text(sql)), call.macro_name.clone())); + } + } + outputs +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs index c63694a6a4..b25a71ef65 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs @@ -130,12 +130,6 @@ fn given_invalid_macro_files_when_parsing_then_each_is_rejected() { expected_code: "P014", expected_message: "macro call @other() in SQL macro m() does not say which kind of macro it calls", }, - DefinitionErrorTestCase { - description: "expression without returns", - source: "MACRO (description \"x\", arguments (a BIGINT));\na + 1", - expected_code: "P015", - expected_message: "expression macro m() needs returns ", - }, DefinitionErrorTestCase { description: "CTE list with returns", source: "MACRO (description \"x\", returns BIGINT);\nWITH a AS (SELECT 1)", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index 6307803193..8b9896b7e3 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -9,7 +9,7 @@ use crate::sql_macros::tests::test_types::{ const CENTS_TO_DOLLARS: (&str, &str) = ( "cents_to_dollars", - "MACRO (\n description \"Cents to dollars\",\n arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n returns DECIMAL(18, 2),\n);\n\nROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places)\n", + "MACRO (\n description \"Cents to dollars\",\n arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n returns DECIMAL(18, 2),\n);\n\nCAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2))\n", ); const NET_AMOUNT: (&str, &str) = ( "net_amount", @@ -121,6 +121,22 @@ const ORDER_KEY: (&str, &str) = ( "order_key", "MACRO (description \"Order key\", arguments (store_code VARCHAR, order_number ANY), returns VARCHAR);\n\n@store_code || '::' || CAST(@order_number AS VARCHAR)\n", ); +const SHOUTED_WRONG: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\nUPPER(@value)\n", +); +const SHOUTED_UNTYPED: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value VARCHAR));\n\nUPPER(@value)\n", +); +const SHOUTED_ANY: (&str, &str) = ( + "shouted", + "MACRO (description \"Shouted\", arguments (value ANY), returns BIGINT);\n\n@value\n", +); +const SHOUTED_NESTED: (&str, &str) = ( + "shouted_twice", + "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\n@sql.plus_one(LENGTH(@value)) || 'x'\n", +); const ODD: (&str, &str) = ( "odd", "MACRO (description \"Odd\", arguments (v FOO BAR), returns BIGINT);\n\n@v + 1\n", @@ -134,21 +150,21 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { adapter: "", macros: &[CENTS_TO_DOLLARS], sql: "SELECT @sql.cents_to_dollars(o.fee_cents + o.tax_cents, places := 0) AS fees FROM orders AS o", - expected_sql: "SELECT ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees FROM orders AS o", + expected_sql: "SELECT CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM orders AS o", }, ExpansionTestCase { description: "expression stacking another expression macro", adapter: "", macros: &[CENTS_TO_DOLLARS, NET_AMOUNT], sql: "SELECT @sql.net_amount(o.gross_cents, o.refund_cents) AS net FROM orders AS o", - expected_sql: "SELECT ROUND(CAST((o.gross_cents - COALESCE(o.refund_cents, 0)) AS DECIMAL(18, 4)) / 100, 2) AS net FROM orders AS o", + expected_sql: "SELECT CAST(ROUND((o.gross_cents - COALESCE(o.refund_cents, 0)) / 100, 2) AS DECIMAL(18, 2)) AS net FROM orders AS o", }, ExpansionTestCase { description: "a backslash-escaped quote in a BigQuery string is not a call", adapter: "bigquery", macros: &[NET_AMOUNT, CENTS_TO_DOLLARS], sql: "SELECT 'it\\'s @sql.x(' AS note, @sql.net_amount(o.a, o.b) AS net FROM orders AS o", - expected_sql: "SELECT 'it\\'s @sql.x(' AS note, ROUND(CAST((o.a - COALESCE(o.b, 0)) AS DECIMAL(18, 4)) / 100, 2) AS net FROM orders AS o", + expected_sql: "SELECT 'it\\'s @sql.x(' AS note, CAST(ROUND((o.a - COALESCE(o.b, 0)) / 100, 2) AS DECIMAL(18, 2)) AS net FROM orders AS o", }, ExpansionTestCase { description: "an expression call before AT TIME ZONE is parenthesised", @@ -281,7 +297,7 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { adapter: "", macros: &[CENTS_TO_DOLLARS], sql: "SELECT @py.label(@sql.cents_to_dollars(o.fee)) AS fee FROM orders AS o", - expected_sql: "SELECT @py.label(\"ROUND(CAST(o.fee AS DECIMAL(18, 4)) / 100, 2)\") AS fee FROM orders AS o", + expected_sql: "SELECT @py.label(\"CAST(ROUND(o.fee / 100, 2) AS DECIMAL(18, 2))\") AS fee FROM orders AS o", }, ExpansionTestCase { description: "runtime ROUND precision where the adapter allows one", @@ -388,6 +404,20 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "SELECT @sql.odd(1) AS a", expected_sql: "SELECT 1 + 1 AS a", }, + ExpansionTestCase { + description: "an expression macro without returns", + adapter: "duckdb", + macros: &[SHOUTED_UNTYPED], + sql: "SELECT @sql.shouted(o.status) AS a FROM orders AS o", + expected_sql: "SELECT UPPER(o.status) AS a FROM orders AS o", + }, + ExpansionTestCase { + description: "an ANY argument leaves the body untyped, so returns is trusted", + adapter: "duckdb", + macros: &[SHOUTED_ANY], + sql: "SELECT @sql.shouted(o.status) AS a FROM orders AS o", + expected_sql: "SELECT o.status AS a FROM orders AS o", + }, ]; for test_case in test_cases { @@ -429,6 +459,22 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { expected_code: "P018", expected_message: "column \"created_at\" from macro audit_columns() clashes with a column already defined in model:orders_clean", }, + ExpansionErrorTestCase { + description: "a body whose known type is not returns", + adapter: "duckdb", + macros: &[SHOUTED_WRONG], + sql: "SELECT @sql.shouted(o.status) FROM orders AS o", + expected_code: "P024", + expected_message: "macro shouted() declares returns BIGINT, but its body is VARCHAR", + }, + ExpansionErrorTestCase { + description: "a body typed through a nested call", + adapter: "duckdb", + macros: &[SHOUTED_NESTED, PLUS_ONE], + sql: "SELECT @sql.shouted_twice(o.status) FROM orders AS o", + expected_code: "P024", + expected_message: "macro shouted_twice() declares returns BIGINT, but its body is VARCHAR", + }, ExpansionErrorTestCase { description: "cycle", adapter: "", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index b655b96bbe..e847311788 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -1,4 +1,5 @@ use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; +use crate::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use crate::sql_macros::models::TypedCall; use crate::sql_macros::tests::helpers::expanded_text_for; use crate::sql_macros::tests::test_types::{TypedAnalysisTestCase, TypedCallTestCase}; @@ -194,3 +195,32 @@ fn given_the_reserved_analysis_function_when_comparing_then_authored_sql_rejects polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION ); } + +#[test] +fn given_calls_without_returns_when_listing_outputs_then_only_whole_named_items_are_named() { + let untyped = |start: usize, end: usize| TypedCall { + start, + end, + macro_name: "shout".to_owned(), + returns: None, + }; + let sql: &str = + "SELECT UPPER(a) AS loud, UPPER(b) quiet, UPPER(c) || 'x' AS joined, UPPER(d) FROM t"; + let calls: Vec = ["UPPER(a)", "UPPER(b)", "UPPER(c)", "UPPER(d)"] + .iter() + .map(|text| { + let start: usize = sql.find(text).unwrap_or_default(); + untyped(start, start + text.len()) + }) + .collect(); + + let outputs: Vec<(String, String)> = untyped_call_outputs(sql, &calls); + + assert_eq!( + outputs, + vec![ + ("loud".to_owned(), "shout".to_owned()), + ("quiet".to_owned(), "shout".to_owned()), + ] + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index d3d532a8c0..b7ef9cdd1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::contracts::models::{ PromotionConflict, PromotionModel, PromotionRequest, }; +use sqlbuild_analysis::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use sqlbuild_analysis::type_system::errors::TypeNormalizationError; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -80,6 +81,7 @@ fn fact_contract_model(model: &CompiledModelFacts) -> ContractModel { unchecked_output_columns: analysis .map(|analysis| analysis.unchecked_output_columns.clone()) .unwrap_or_default(), + untyped_call_outputs: untyped_call_outputs(&model.query_sql, &model.typed_calls), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index 2906432507..168775c6d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::sql_macros::constants::{ - BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, MIXED_KIND_CODE, + BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, MIXED_KIND_CODE, RETURNS_MISMATCH_CODE, }; use sqlbuild_analysis::sql_macros::errors::ExpansionFailure; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; @@ -149,6 +149,8 @@ pub(crate) fn expanded_sql_macro_calls<'py>( Err(ExpansionFailure::Host(error)) => return Err(error), Err(ExpansionFailure::Macro(error)) => { let located: SqlMacroError = match host.call_location((call.start, call.end)) { + // A wrong returns is the macro file's error; the call comes after it. + Some(location) if error.code == RETURNS_MISMATCH_CODE => error.at(location), Some(location) => error.at_call(location), None => error, }; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs index 7765ad08c9..edb42e2cb1 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs @@ -187,13 +187,12 @@ impl OutputRule<'_> { .cast_type .as_ref() .is_some_and(|data_type| type_spelling_equal(data_type, &column.data_type)); + let call: Option<&TypedCall> = (!explicit_cast) + .then(|| self.whole_call(expression)) + .flatten(); let passes = cast_matches_contract || column.type_proven && direct && !scope.set_branch - || column.type_proven - && !explicit_cast - && self - .whole_call(expression) - .is_some_and(|call| call.returns.is_some()); + || column.type_proven && call.is_some_and(|call| call.returns.is_some()); if passes { continue; } @@ -213,10 +212,16 @@ impl OutputRule<'_> { "output {:?} {reason}; declared type is {}", name, column.data_type ), - format!( - "Wrap the complete expression in CAST(... AS {}) or establish that exact type on a direct upstream column.", - column.data_type - ), + match call.filter(|call| call.returns.is_none()) { + Some(call) => format!( + "Declare returns {} in SQL macro {}(), or wrap the call in CAST(... AS {}).", + column.data_type, call.macro_name, column.data_type + ), + None => format!( + "Wrap the complete expression in CAST(... AS {}) or establish that exact type on a direct upstream column.", + column.data_type + ), + }, ); } } diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index c030d8865e..ad85b6351a 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -786,6 +786,17 @@ def _sql_macro_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="sql-macro-returns-disagrees-with-body", + expected_code="P024", + expected_message="macro x() declares returns DOUBLE, but its body is VARCHAR", + files=_sql_macro_columns( + columns="@sql.x(amount) AS amount, status", + files=_x_macro( + arguments="v DOUBLE", returns=", returns DOUBLE", body="CAST(@v AS VARCHAR)\n" + ), + ), + ), failure_case( name="sql-macro-body-declaration-not-visible", expected_code="S026", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index af4b3b125a..288116e46c 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -564,6 +564,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "P021", "P022", "P023", + "P024", "S026", } ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index 01e92ccaca..aca14a3337 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -52,7 +52,7 @@ 'MACRO (\n description "Cents to dollars",\n' " arguments (amount_cents BIGINT, places INT CONST DEFAULT 2),\n" " returns DECIMAL(18, 2),\n);\n\n" - "ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places)\n" + "CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2))\n" ) } NET_AMOUNT: dict[str, str] = { @@ -232,13 +232,16 @@ def sql_text_macros(folder: str, names: list[str]) -> dict[str, str]: } -def text_macro(name: str, returns: str, body: str) -> dict[str, str]: - """The SQL macro `name` taking one VARCHAR value and declaring `returns`.""" +def text_macro( + name: str, returns: str | None, body: str, argument: str = "VARCHAR" +) -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value and declaring `returns`, if any.""" + declared: str = "" if returns is None else f" returns {returns},\n" return { f"macros/sql/{name}.sql": ( - f'MACRO (\n description "Text from a value",\n arguments (value VARCHAR),\n' - f" returns {returns},\n);\n\n{body}\n" + f'MACRO (\n description "Text from a value",\n arguments (value {argument}),\n' + f"{declared});\n\n{body}\n" ) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py index fea2ea77b5..7ae41d98ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py @@ -3,6 +3,7 @@ from __future__ import annotations import subprocess +from decimal import Decimal from pathlib import Path import pytest @@ -145,7 +146,7 @@ ), }, query="SELECT order_id, amount, fees, net, label FROM orders", - expected_rows=[(2, 5.0, 1.0, 4.5, "[5.0]")], + expected_rows=[(2, Decimal("5.00"), Decimal("1.00"), Decimal("4.50"), "[5.00]")], ), SqlMacroBuildTestCase( description="cte_list_and_column_list_macros_with_enum_and_list_constant_arguments", @@ -458,7 +459,7 @@ def test_given_any_spelling_of_list_calls_when_building_then_the_model_holds_the files={ **CENTS_TO_DOLLARS, "macros/sql/zz_invalid.sql": ( - 'MACRO (description "Invalid", arguments (v BIGINT));\n\n@v + 1\n' + 'MACRO (description "Invalid", returns BIGINT);\n\nWITH a AS (SELECT 1)\n' ), **orders_model( 'SELECT @sql.cents_to_dollars(amount_cents) AS d FROM __ref("stg_orders")\n' @@ -468,7 +469,7 @@ def test_given_any_spelling_of_list_calls_when_building_then_the_model_holds_the expected_returncode=1, expected_fragments=( "Resource: macro:cents_to_dollars", - "ERROR S021 macros/sql/zz_invalid.sql: expression macro zz_invalid() needs returns", + "ERROR S021 macros/sql/zz_invalid.sql: CTE-list macro zz_invalid() cannot declare returns", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py index e691bff1e8..0e4a440594 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py @@ -339,13 +339,19 @@ ), ), SqlMacroErrorTestCase( - description="expression_macro_without_returns", + description="returns_that_disagrees_with_the_body_is_reported_at_the_macro_file", files={ - "macros/sql/x.sql": _X_HEADER.format(args="v BIGINT", returns="") + "@v + 1\n", - **_CALL_X, + "macros/sql/x.sql": _X_HEADER.format(args="v VARCHAR", returns=", returns BIGINT") + + "UPPER(@v)\n", + **orders_model("SELECT @sql.x('a') AS a\n"), }, - expected_code="P015", - expected_fragments=("expression macro x() needs returns in its MACRO header",), + expected_code="P024", + expected_fragments=( + "macro x() declares returns BIGINT, but its body is VARCHAR\n" + " model: orders\n" + " --> macros/sql/x.sql:3:1", + "declare returns VARCHAR, CAST the body to BIGINT, or remove returns", + ), ), SqlMacroErrorTestCase( description="untyped_argument", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py new file mode 100644 index 0000000000..6600e400cd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -0,0 +1,142 @@ +"""E2E coverage for optional SQL macro returns types and the check of returns against the body.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + contract_model, + output, + run_project, + text_macro, + with_rules, +) + +_UNTYPED_UPSTREAM: dict[str, str] = { + "models/marts/up.sql": ( + 'MODEL (description "Up");\n\nSELECT NULL AS s FROM __ref("stg_orders") AS o\n' + ), +} +_UNTYPED_CALL: str = 'SELECT @sql.m(u.s) AS v FROM __ref("up") AS u\n' + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_macro_without_returns_is_typed_from_its_body", + files={ + **text_macro("m", None, "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_macro_without_returns_keeps_a_known_body_type_that_disagrees", + files={ + **text_macro("m", None, "LENGTH(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'v' inferred as BIGINT",), + ), + SqlMacroCompileTestCase( + description="a_macro_with_matching_returns_compiles", + files={ + **text_macro("m", "VARCHAR", "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_wrong_returns_is_reported_at_the_macro_file", + files={ + **text_macro("m", "BIGINT", "UPPER(@value)"), + **contract_model( + "orders", + "v (type BIGINT)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P024]: macro m() declares returns BIGINT, but its body is VARCHAR\n" + " model: orders\n" + " --> macros/sql/m.sql:7:1", + ), + ), + SqlMacroCompileTestCase( + description="an_any_argument_leaves_the_body_unknown_so_returns_is_trusted", + files={ + **with_rules("SQBRCONTRACT105"), + **text_macro("m", "BIGINT", "@value", argument="ANY"), + **_UNTYPED_UPSTREAM, + **contract_model("orders", "v (type BIGINT)", _UNTYPED_CALL), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_untyped_call_without_returns_suggests_returns_on_k003", + files={ + **text_macro("m", None, "@value", argument="ANY"), + **_UNTYPED_UPSTREAM, + **contract_model("orders", "v (type BIGINT)", _UNTYPED_CALL), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared BIGINT", + "declare returns BIGINT in SQL macro m(), or add an explicit CAST", + ), + ), + SqlMacroCompileTestCase( + description="an_untyped_call_without_returns_suggests_returns_on_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **text_macro("m", None, "UPPER(@value)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + 'error[SQBRCONTRACT105]: output "v" is calculated without an outer explicit cast', + "Declare returns VARCHAR in SQL macro m(), or wrap the call in CAST(... AS VARCHAR).", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_optional_returns_when_compiling_then_calls_are_typed_and_checked( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + for fragment in test_case.unexpected_fragments: + assert fragment not in output(result), output(result) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py index d5bfa88f92..6d837cd889 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_types.py @@ -222,7 +222,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( unexpected_fragments=("K003",), ), SqlMacroCompileTestCase( - description="a_time_body_keeps_its_inferred_type", + description="a_time_body_disagrees_with_returns_at_the_macro_file", files={ **text_macro("m", "VARCHAR", "CAST('10:00:00' AS TIME)"), **contract_model( @@ -232,10 +232,12 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'v' inferred as",), + expected_fragments=( + "error[P024]: macro m() declares returns VARCHAR, but its body is TIME", + ), ), SqlMacroCompileTestCase( - description="an_array_body_keeps_its_inferred_type", + description="an_array_body_disagrees_with_returns_at_the_macro_file", files={ **text_macro("m", "VARCHAR", "[@value]"), **contract_model( @@ -245,7 +247,9 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'v' inferred as",), + expected_fragments=( + "error[P024]: macro m() declares returns VARCHAR, but its body is", + ), ), SqlMacroCompileTestCase( description="a_longer_text_body_keeps_its_inferred_type", @@ -406,7 +410,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( description="an_untyped_upstream_column_in_a_cte_takes_returns", files={ **with_rules("SQBRCONTRACT105"), - **text_macro("m", "BIGINT", "@value"), + **text_macro("m", "BIGINT", "@value", argument="ANY"), "models/marts/up.sql": ( 'MODEL (description "Up");\n\nSELECT NULL AS s FROM __ref("stg_orders") AS o\n' ), @@ -438,7 +442,7 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( unexpected_fragments=_UNPROVEN, ), SqlMacroCompileTestCase( - description="an_inferred_type_of_another_family_still_disagrees_through_a_cte", + description="an_inferred_type_of_another_family_disagrees_at_the_macro_file", files={ **text_macro("shouted", "BIGINT", "UPPER(@value)"), **contract_model( @@ -450,7 +454,9 @@ def test_given_an_any_argument_misused_when_compiling_then_it_fails( ), }, expected_returncode=1, - expected_fragments=("error[K002]: column 'shouted' inferred as",), + expected_fragments=( + "error[P024]: macro shouted() declares returns BIGINT, but its body is VARCHAR", + ), ), ], ids=lambda case: case.description, From 1af53fe2e7670c03576118843902c21e0fd8da3c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:38:13 +0100 Subject: [PATCH 06/28] feat: name the CTEs of a CTE-list macro call with __cte A CTE-list macro call may pass __cte := name. The macro's last CTE then takes that name and its other CTEs become __, renamed with their references by parsing; arguments are never renamed. Calls without __cte behave as before, and a repeated call now suggests __cte. The formatter also accepts comma-less CTE-list calls in a row. --- .../src/sql_macros/_helpers/cte_rename.rs | 248 ++++++++++++++++++ .../src/sql_macros/_helpers/expansion.rs | 39 ++- .../src/sql_macros/_helpers/mod.rs | 1 + .../src/sql_macros/constants.rs | 7 +- .../src/sql_macros/tests/test_expansion.rs | 53 ++++ .../src/sql_lint/_helpers/formatter_syntax.rs | 4 +- .../src/sql_lint/tests/test_layout_rules.rs | 7 + .../sql_macros/test_sql_macro_cte_names.py | 223 ++++++++++++++++ 8 files changed, 571 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs new file mode 100644 index 0000000000..1e1a082e1c --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs @@ -0,0 +1,248 @@ +//! A CTE-list macro body's CTEs renamed for one call's `__cte` name, located by parsing. + +use polyglot_sql::expressions::Identifier; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; + +use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parameters::{ + ParameterReference, parameter_references, placeholder, with_placeholders, +}; +use crate::sql_macros::_helpers::typed_calls::spliced; +use crate::sql_macros::constants::{ARGUMENT_CODE, CTE_ARGUMENT, CTE_NAME_SEPARATOR}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ + CallArgument, ExpandedText, SqlMacroDefinition, SqlMacroForm, TextSplice, +}; + +/// Dialects tried in turn to parse a body for its CTE names. +const PARSE_DIALECTS: [DialectType; 4] = [ + DialectType::Generic, + DialectType::DuckDB, + DialectType::Snowflake, + DialectType::BigQuery, +]; +/// The query a CTE list is completed with to parse it. +const CTE_SUFFIX: &str = "\nSELECT 1"; + +/// The call's `__cte` name, removed from its arguments, or why it cannot be used. +pub(crate) fn take_cte_argument( + definition: &SqlMacroDefinition, + arguments: Vec, +) -> Result<(Option, Vec), SqlMacroError> { + let mut name: Option = None; + let mut kept: Vec = Vec::with_capacity(arguments.len()); + for argument in arguments { + if !argument + .name + .as_deref() + .is_some_and(|named| named.eq_ignore_ascii_case(CTE_ARGUMENT)) + { + kept.push(argument); + continue; + } + if definition.form != SqlMacroForm::CteList { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} names the CTEs a CTE-list macro inserts; {}() is {} {} macro", + definition.name, + if definition.form == SqlMacroForm::Expression { + "an" + } else { + "a" + }, + definition.form.label() + ), + ) + .helped(format!("remove {CTE_ARGUMENT} := {}", argument.text.trim()))); + } + if name.is_some() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} is passed twice in a call to macro {}()", + definition.name + ), + )); + } + let text: &str = argument.text.trim(); + if !is_bare_identifier(text) { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "{CTE_ARGUMENT} of macro {}() must be a bare CTE name; got {text}", + definition.name + ), + ) + .helped(format!("pass a name such as {CTE_ARGUMENT} := eu_orders"))); + } + name = Some(text.to_owned()); + } + Ok((name, kept)) +} + +/// Whether `text` is one unquoted identifier. +fn is_bare_identifier(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// The name each of a body's CTEs takes: the last takes `name`, the others `__`. +fn renamed(ctes: &[String], name: &str, cte: &str) -> Option { + let index: usize = ctes + .iter() + .position(|defined| defined.eq_ignore_ascii_case(cte))?; + Some(if index + 1 == ctes.len() { + name.to_owned() + } else { + format!("{name}{CTE_NAME_SEPARATOR}{}", ctes[index]) + }) +} + +/// `body` with every CTE it defines, and every reference to one, renamed for `name`. +pub(crate) fn renamed_ctes(body: &ExpandedText, name: &str) -> Result { + let sql: &str = &body.sql; + let scanned: Vec = + tokens(sql).map_err(|_| "it has an unclosed quote or comment".to_owned())?; + let references: Vec = parameter_references(sql, &scanned); + let parsed_text: String = format!("{}{CTE_SUFFIX}", with_placeholders(sql, &references)); + let statement: Expression = parsed(&parsed_text)?; + let Expression::Select(select) = &statement else { + return Err("it is not a WITH list".to_owned()); + }; + let Some(with) = &select.with else { + return Err("it is not a WITH list".to_owned()); + }; + let ctes: Vec = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); + let aliases: Vec = statement + .dfs() + .filter_map(|node| match node { + Expression::Table(table) => table.alias.as_ref().map(|alias| alias.name.clone()), + _ => None, + }) + .collect(); + let mut identifiers: Vec<&Identifier> = with.ctes.iter().map(|cte| &cte.alias).collect(); + for node in statement.dfs() { + match node { + Expression::Table(table) if table.schema.is_none() && table.catalog.is_none() => { + identifiers.push(&table.name); + } + Expression::Column(column) => identifiers.extend(column.table.as_ref()), + Expression::Star(star) => identifiers.extend(star.table.as_ref()), + _ => {} + } + } + let shadowed = |identifier: &Identifier| { + aliases + .iter() + .any(|alias| alias.eq_ignore_ascii_case(&identifier.name)) + }; + let offsets: CharOffsets = CharOffsets::new(&parsed_text); + let mut edits: Vec<(usize, usize, String)> = Vec::new(); + for (position, identifier) in identifiers.into_iter().enumerate() { + let is_definition: bool = position < with.ctes.len(); + let Some(new_name) = renamed(&ctes, name, &identifier.name) else { + continue; + }; + if !is_definition && shadowed(identifier) { + continue; + } + let Some(span) = identifier.span else { + continue; + }; + let start: usize = original_offset(offsets.byte(span.start), sql, &references); + let end: usize = original_offset(offsets.byte(span.end), sql, &references); + let text: String = if identifier.quoted { + format!("\"{new_name}\"") + } else { + new_name + }; + edits.push((start, end, text)); + } + edits.sort_by_key(|(start, _, _)| *start); + edits.dedup_by_key(|(start, _, _)| *start); + let mut renamed_sql: String = String::with_capacity(sql.len() + edits.len() * name.len()); + let mut splices: Vec = Vec::with_capacity(edits.len()); + let mut cursor: usize = 0; + for (start, end, text) in edits { + if start < cursor || end > sql.len() { + continue; + } + renamed_sql.push_str(&sql[cursor..start]); + let output: usize = renamed_sql.len(); + renamed_sql.push_str(&text); + splices.push(TextSplice { + source: (start, end), + output: (output, renamed_sql.len()), + }); + cursor = end; + } + renamed_sql.push_str(&sql[cursor..]); + Ok(ExpandedText { + calls: spliced(body.calls.clone(), &splices), + sql: renamed_sql, + }) +} + +/// The one statement `text` parses to, in the first dialect that parses it. +fn parsed(text: &str) -> Result { + let mut first_error: Option = None; + for dialect in PARSE_DIALECTS { + match polyglot_sql::parse(text, dialect) { + Ok(mut statements) if statements.len() == 1 => { + if let Some(statement) = statements.pop() { + return Ok(statement); + } + } + Ok(_) => { + first_error.get_or_insert_with(|| "it holds more than one statement".to_owned()); + } + Err(error) => { + first_error.get_or_insert_with(|| error.to_string()); + } + } + } + Err(first_error.unwrap_or_default()) +} + +/// The byte offset in `sql` of byte `offset` of `sql` with its references as placeholders. +fn original_offset(offset: usize, sql: &str, references: &[ParameterReference]) -> usize { + let mut moved: isize = 0; + for reference in references { + let placed_start: isize = reference.start as isize + moved; + if (offset as isize) <= placed_start { + break; + } + moved += placeholder(&reference.name).len() as isize + - (reference.end - reference.start) as isize; + } + usize::try_from(offset as isize - moved) + .unwrap_or(0) + .min(sql.len()) +} + +/// Byte offsets of a text's character offsets. +struct CharOffsets { + bytes: Option>, +} + +impl CharOffsets { + fn new(text: &str) -> Self { + if text.is_ascii() { + return Self { bytes: None }; + } + let mut bytes: Vec = text.char_indices().map(|(offset, _)| offset).collect(); + bytes.push(text.len()); + Self { bytes: Some(bytes) } + } + + fn byte(&self, character: usize) -> usize { + match &self.bytes { + Some(bytes) => bytes.get(character).copied().unwrap_or(character), + None => character, + } + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index c452652b41..0900cc3aa1 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -7,6 +7,7 @@ use crate::sql_macros::_helpers::arguments::{ ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, }; use crate::sql_macros::_helpers::body::{expand_body, with_line_comment_closed}; +use crate::sql_macros::_helpers::cte_rename::{renamed_ctes, take_cte_argument}; use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs, outer_sql_calls}; use crate::sql_macros::_helpers::lexing::{Token, line_column, tokens}; use crate::sql_macros::_helpers::literal_positions::{ @@ -18,16 +19,16 @@ use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; use crate::sql_macros::_helpers::returns_check::check_body_returns; use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ - ARGUMENT_CODE, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, MISPLACED_CALL_CODE, - NAME_CLASH_CODE, + ARGUMENT_CODE, CTE_ARGUMENT, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, + MISPLACED_CALL_CODE, NAME_CLASH_CODE, }; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallPosition, - CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, RenderedDeclarations, - SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TextContext, - TypedCall, + ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallArgument, + CallPosition, CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, + RenderedDeclarations, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, + SqlMacroLocation, TextContext, TypedCall, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -188,8 +189,11 @@ pub(crate) fn expand_call( } check_adapter_rules(&definition, host.adapter())?; let arguments_text: &str = &context.text[call.arguments.0..call.arguments.1]; - let bound: Vec = split_arguments(arguments_text) - .and_then(|arguments| bind_arguments(&definition, &arguments)) + let (cte_name, arguments): (Option, Vec) = + split_arguments(arguments_text) + .and_then(|arguments| take_cte_argument(&definition, arguments)) + .map_err(|error| error.at(body_location(&definition)))?; + let bound: Vec = bind_arguments(&definition, &arguments) .map_err(|error| error.at(body_location(&definition)))?; let macro_scope: SqlMacroScope = SqlMacroScope::Macro(Arc::clone(&definition)); let mut rendered: Vec = Vec::with_capacity(bound.len()); @@ -218,6 +222,19 @@ pub(crate) fn expand_call( }, ) .map_err(|failure| located(failure, &body_location(&definition)))?; + let body: ExpandedText = match &cte_name { + Some(name) => renamed_ctes(&body, name).map_err(|reason| { + SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!( + "body of macro {}() is not a valid CTE-list macro: {reason}", + definition.name + ), + ) + .at(body_location(&definition)) + })?, + None => body, + }; check_body_returns( &definition, &body, @@ -498,6 +515,12 @@ fn check_clashes( let clash_location: SqlMacroLocation = body_offset_location(definition, definition.body.find(name.as_str()).unwrap_or(0)); let lists_with = |plural: &str| { + if definition.form == SqlMacroForm::CteList { + return format!( + "pass {CTE_ARGUMENT} := to give the CTEs one call inserts their own names, as in @sql.{}(..., {CTE_ARGUMENT} := first_orders), or rename one of the {plural}", + definition.name + ); + } format!( "rename one of the {plural}; a model can't call a {} macro whose {noun} names it already uses", definition.form.label() diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs index 3296e7685d..f9a3154ce9 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod arguments; pub(crate) mod body; +pub(crate) mod cte_rename; pub(crate) mod definition; pub(crate) mod expansion; pub(crate) mod header; diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 7fb6c554c4..7042c9de5c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -41,7 +41,12 @@ pub(crate) const ENUM_KEYWORD: &str = "ENUM"; pub(crate) const LIST_KEYWORD: &str = "LIST"; pub(crate) const SET_KEYWORD: &str = "SET"; /// Names a parameter may not take, because `@name` already means something else. -pub(crate) const RESERVED_PARAMETER_NAMES: [&str; 5] = ["sql", "py", "const", "enum", "param"]; +pub(crate) const RESERVED_PARAMETER_NAMES: [&str; 6] = + ["sql", "py", "const", "enum", "param", CTE_ARGUMENT]; +/// The reserved call argument naming the CTEs a CTE-list macro call inserts. +pub const CTE_ARGUMENT: &str = "__cte"; +/// What joins a call's `__cte` name to the name of each CTE but the last. +pub(crate) const CTE_NAME_SEPARATOR: &str = "__"; /// Type names that would make a parameter raw, untyped text. pub(crate) const RAW_TYPE_NAMES: [&str; 3] = ["RAW", "SQL", "EXPRESSION"]; /// The type of a parameter that takes any scalar expression and checks no type. diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index 8b9896b7e3..d057c1d072 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -208,6 +208,27 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\n@sql.clean_orders(orders_raw)\nfinal AS (SELECT order_id, amount FROM latest)\nSELECT * FROM final", expected_sql: "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\nlive AS (SELECT * FROM orders_raw WHERE NOT is_deleted),\nranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM live\n),\nlatest AS (SELECT * EXCLUDE (row_num) FROM ranked WHERE row_num = 1),\nfinal AS (SELECT order_id, amount FROM latest)\nSELECT * FROM final", }, + ExpansionTestCase { + description: "CTE list called twice, each call naming its CTEs with __cte", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH eu_raw AS (SELECT 1 AS order_id), us_raw AS (SELECT 2 AS order_id),\n@sql.clean_orders(eu_raw, __cte := eu_orders)\n@sql.clean_orders(us_raw, __CTE := us_orders)\nfinal AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders)\nSELECT * FROM final", + expected_sql: "WITH eu_raw AS (SELECT 1 AS order_id), us_raw AS (SELECT 2 AS order_id),\neu_orders__live AS (SELECT * FROM eu_raw WHERE NOT is_deleted),\neu_orders__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM eu_orders__live\n),\neu_orders AS (SELECT * EXCLUDE (row_num) FROM eu_orders__ranked WHERE row_num = 1),\nus_orders__live AS (SELECT * FROM us_raw WHERE NOT is_deleted),\nus_orders__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM us_orders__live\n),\nus_orders AS (SELECT * EXCLUDE (row_num) FROM us_orders__ranked WHERE row_num = 1),\nfinal AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders)\nSELECT * FROM final", + }, + ExpansionTestCase { + description: "__cte renames the macro's CTEs but never an argument of the same name", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH live AS (SELECT 1 AS order_id), @sql.clean_orders(live, __cte := picked) SELECT * FROM picked", + expected_sql: "WITH live AS (SELECT 1 AS order_id), picked__live AS (SELECT * FROM live WHERE NOT is_deleted),\npicked__ranked AS (\n SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY updated_at DESC) AS row_num\n FROM picked__live\n),\npicked AS (SELECT * EXCLUDE (row_num) FROM picked__ranked WHERE row_num = 1) SELECT * FROM picked", + }, + ExpansionTestCase { + description: "__cte renames qualified references to a renamed CTE", + adapter: "", + macros: &[LATEST_PER_KEY], + sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), @sql.latest_per_key(r, id, __cte := l) SELECT * FROM l", + expected_sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), l__latest_ranked AS (\n SELECT r.*, ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) AS row_num FROM r\n),\nl AS (SELECT * EXCLUDE (row_num) FROM l__latest_ranked WHERE row_num = 1) SELECT * FROM l", + }, ExpansionTestCase { description: "CTE list extracted from a model compiles to the original", adapter: "", @@ -451,6 +472,38 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { expected_code: "P018", expected_message: "CTE \"live\" from macro clean_orders() clashes with a CTE from macro clean_orders() in model:orders_clean", }, + ExpansionErrorTestCase { + description: "__cte on an expression macro", + adapter: "", + macros: &[PLUS_ONE], + sql: "SELECT @sql.plus_one(1, __cte := x)", + expected_code: "P017", + expected_message: "__cte names the CTEs a CTE-list macro inserts; plus_one() is an expression macro", + }, + ExpansionErrorTestCase { + description: "__cte that is not a bare name", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), @sql.clean_orders(a, __cte := 'x') SELECT 1", + expected_code: "P017", + expected_message: "__cte of macro clean_orders() must be a bare CTE name; got 'x'", + }, + ExpansionErrorTestCase { + description: "the same __cte name twice", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), @sql.clean_orders(a, __cte := x) @sql.clean_orders(a, __cte := x) SELECT 1", + expected_code: "P018", + expected_message: "CTE \"x__live\" from macro clean_orders() clashes with a CTE from macro clean_orders() in model:orders_clean", + }, + ExpansionErrorTestCase { + description: "a __cte name the caller already defines", + adapter: "", + macros: &[CLEAN_ORDERS], + sql: "WITH a AS (SELECT 1), x AS (SELECT 2), @sql.clean_orders(a, __cte := x) SELECT 1", + expected_code: "P018", + expected_message: "CTE \"x\" from macro clean_orders() clashes with a CTE already defined in model:orders_clean", + }, ExpansionErrorTestCase { description: "column clash with the caller", adapter: "", diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs index a1af8326ea..f99ce7bcaa 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter_syntax.rs @@ -295,10 +295,10 @@ pub(super) fn cte_macro_indices(tokens: &[Token]) -> HashSet { } else if token.text.starts_with(LINT_SENTINEL_PREFIX) && scopes.contains(&depths[index]) && index > 0 - && matches!( + && (matches!( tokens[index - 1].token_type, TokenType::With | TokenType::Comma - ) + ) || indices.contains(&(index - 1))) && tokens.get(index + 1).is_some_and(|next| { matches!(next.token_type, TokenType::Comma | TokenType::Select) || starts_cte(tokens, index + 1) diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs index b5d068a929..b748af51a2 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_layout_rules.rs @@ -169,6 +169,13 @@ fn given_clause_layout_rules_when_formatting_then_each_rule_applies_idempotently line_width: 100, expected_sql: "WITH\n__sqb_lint_0__,\n\npaid AS (\n SELECT *\n FROM orders\n)\n\nSELECT *\nFROM paid", }, + test_types::LayoutRuleTestCase { + description: "comma-less CTE macro placeholders in a row are separated like CTEs", + dialect: "duckdb", + sql: "WITH a AS (SELECT 1 AS x), __sqb_lint_0__ __sqb_lint_1__ paid AS (SELECT * FROM a) SELECT * FROM paid", + line_width: 100, + expected_sql: "WITH a AS (\n SELECT 1 AS x\n),\n\n__sqb_lint_0__\n\n__sqb_lint_1__\npaid AS (\n SELECT *\n FROM a\n)\nSELECT *\nFROM paid", + }, test_types::LayoutRuleTestCase { description: "an alias and comma that push a call over the width break the call", dialect: "snowflake", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py new file mode 100644 index 0000000000..ce3b2dc545 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py @@ -0,0 +1,223 @@ +"""E2E coverage for the `__cte` argument naming the CTEs a CTE-list SQL macro call inserts.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroBuildTestCase, + SqlMacroCommandTestCase, + SqlMacroErrorTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + ALL_MACROS, + CLEAN_ORDERS, + DATABASE, + contract_model, + inspected_text, + orders_model, + output, + run_project, + text_macro, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb + +_TWO_CALLS: str = ( + "WITH gb_raw AS (SELECT * FROM __ref(\"stg_orders\") WHERE country_code = 'GB'),\n" + "fr_raw AS (SELECT * FROM __ref(\"stg_orders\") WHERE country_code = 'FR'),\n" + "@sql.clean_orders(gb_raw, __cte := gb_orders)\n" + "@sql.clean_orders(fr_raw, __cte := fr_orders)\n" + "final AS (\n" + " SELECT order_id, amount_cents FROM gb_orders\n" + " UNION ALL SELECT order_id, amount_cents FROM fr_orders\n" + ")\nSELECT * FROM final\n" +) +_INTERMEDIATE: str = ( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.clean_orders(raw, __cte := cleaned)\n" + "final AS (SELECT COUNT(*) AS live_rows FROM cleaned__live)\nSELECT * FROM final\n" +) +_CTE_LIST_TEST: str = ( + 'TEST (mode macro, name "clean_orders_named_twice");\n\nWITH\n' + "input_orders AS (\n" + " SELECT 1 AS order_id, TIMESTAMP '2026-01-01' AS updated_at, FALSE AS is_deleted\n" + " UNION ALL SELECT 1, TIMESTAMP '2026-01-02', FALSE\n),\n" + "@sql.clean_orders(input_orders, __cte := first_pass)\n" + "@sql.clean_orders(first_pass, __cte := second_pass)\n" + "__macro_actual__ AS (SELECT order_id, updated_at FROM second_pass),\n" + "__macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroBuildTestCase( + description="the_same_cte_list_macro_called_twice_with_cte_names", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + query="SELECT order_id, amount_cents FROM orders ORDER BY order_id", + expected_rows=[(1, 1300), (2, 500)], + ), + SqlMacroBuildTestCase( + description="a_caller_reads_a_prefixed_intermediate_cte", + files={**CLEAN_ORDERS, **orders_model(_INTERMEDIATE)}, + query="SELECT live_rows FROM orders", + expected_rows=[(3,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_cte_names_when_building_then_each_call_inserts_its_own_ctes( + test_case: SqlMacroBuildTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project(tmp_path=tmp_path, files=test_case.files, command=("build",)) + + assert result.returncode == 0, output(result) + assert ( + query_duckdb(db_path=project_dir / DATABASE, sql=test_case.query) == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroErrorTestCase( + description="cte_on_an_expression_macro", + files={ + **text_macro("m", "VARCHAR", "UPPER(@value)"), + **orders_model('SELECT @sql.m(status, __cte := x) AS v FROM __ref("stg_orders")\n'), + }, + expected_code="P017", + expected_fragments=( + "__cte names the CTEs a CTE-list macro inserts; m() is an expression macro", + ), + ), + SqlMacroErrorTestCase( + description="cte_that_is_not_a_bare_name", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + '@sql.clean_orders(raw, __cte := "gb orders")\nSELECT * FROM raw\n' + ), + }, + expected_code="P017", + expected_fragments=( + '__cte of macro clean_orders() must be a bare CTE name; got "gb orders"', + ), + ), + SqlMacroErrorTestCase( + description="the_same_macro_twice_without_cte_suggests_it", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.clean_orders(raw)\n@sql.clean_orders(raw)\nSELECT * FROM latest\n" + ), + }, + expected_code="P018", + expected_fragments=( + 'CTE "live" from macro clean_orders() clashes with a CTE from macro clean_orders()', + "pass __cte := to give the CTEs one call inserts their own names", + ), + ), + SqlMacroErrorTestCase( + description="a_cte_name_the_caller_already_defines", + files={ + **CLEAN_ORDERS, + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "picked AS (SELECT 1 AS x),\n" + "@sql.clean_orders(raw, __cte := picked)\nSELECT * FROM picked\n" + ), + }, + expected_code="P018", + expected_fragments=( + 'CTE "picked" from macro clean_orders() clashes with a CTE already defined', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_invalid_cte_name_when_compiling_then_the_error_names_the_problem( + test_case: SqlMacroErrorTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == 1, output(result) + assert f"error[{test_case.expected_code}]" in output(result), output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCommandTestCase( + description="macro_mode_test_with_cte_names", + files={ + **ALL_MACROS, + **orders_model(_TWO_CALLS), + "tests/unit/test_clean_orders.sql": _CTE_LIST_TEST, + }, + command=("test",), + expected_returncode=0, + expected_fragments=("PASS=1", "clean_orders_named_twice"), + ), + SqlMacroCommandTestCase( + description="lineage_reads_through_renamed_ctes", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("lineage", "orders.amount_cents", "--format", "list"), + expected_returncode=0, + expected_fragments=("stg_orders.amount_cents -> orders.amount_cents",), + ), + SqlMacroCommandTestCase( + description="contracts_type_columns_through_renamed_ctes", + files={ + **CLEAN_ORDERS, + **contract_model( + "orders", "order_id (type BIGINT), amount_cents (type BIGINT)", _TWO_CALLS + ), + }, + command=("compile", "--no-cache"), + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCommandTestCase( + description="format_keeps_the_cte_argument", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("format",), + expected_returncode=0, + expected_fragments=( + "@sql.clean_orders(gb_raw, __cte := gb_orders)", + "@sql.clean_orders(fr_raw, __cte := fr_orders)", + ), + inspected_paths=("models/marts/orders.sql",), + ), + SqlMacroCommandTestCase( + description="lint_sees_renamed_ctes_as_hand_written", + files={**CLEAN_ORDERS, **orders_model(_TWO_CALLS)}, + command=("check",), + expected_returncode=0, + expected_fragments=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_cte_names_when_running_a_command_then_renamed_ctes_act_as_written( + test_case: SqlMacroCommandTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=test_case.command + ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed From e97e7a03f0568397489d1d05412281e502211b96 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:18:55 +0100 Subject: [PATCH 07/28] feat: add returns TYPE OF and ANY type families to SQL macros returns TYPE OF types each call as its argument through the typed-like analysis marker, where the body's own type is unknown. An ANY argument may be narrowed to ANY NUMERIC, ANY TEXT or ANY TEMPORAL: calls are checked against the family at the argument, and the body is checked at the macro file with the family's widest member (P025). Bumps polyglot to the typed-like marker. --- Cargo.lock | 6 +- Cargo.toml | 4 +- .../analysis_session/_helpers/cte_facts.rs | 21 +- .../analysis_session/_helpers/dict_walk.rs | 15 +- .../_helpers/sql_macro_parts.rs | 5 +- .../main/sql_macro_definition_digest.rs | 10 +- .../_helpers/compatibility_types.rs | 26 ++ .../src/sql_macros/_helpers/arguments.rs | 31 +- .../src/sql_macros/_helpers/body.rs | 5 +- .../src/sql_macros/_helpers/definition.rs | 63 +++- .../src/sql_macros/_helpers/expansion.rs | 36 +- .../src/sql_macros/_helpers/header.rs | 44 ++- .../sql_macros/_helpers/literal_positions.rs | 6 +- .../src/sql_macros/_helpers/probes.rs | 26 +- .../src/sql_macros/_helpers/returns_check.rs | 113 +++++- .../src/sql_macros/_helpers/typed_calls.rs | 15 +- .../src/sql_macros/constants.rs | 7 + .../sql_macros/main/untyped_call_outputs.rs | 5 +- .../src/sql_macros/models.rs | 78 ++++- .../src/sql_macros/tests/test_definitions.rs | 30 ++ .../src/sql_macros/tests/test_expansion.rs | 31 ++ .../src/sql_macros/tests/test_typed_calls.rs | 28 +- .../_helpers/render/sql_macro_host.rs | 10 +- .../rules/_helpers/explicit_output_types.rs | 6 +- .../_helpers/corpus/render_failure_cases.py | 9 + scripts/compiler_differential/constants.py | 1 + .../sql_macros/test_sql_macro_returns.py | 324 ++++++++++++++++++ .../cli/commands/main/test/test_since.py | 36 ++ 28 files changed, 921 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a755186267..bd814d663c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" +source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" +source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89#dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89" +source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 41c608b4cc..81d7282542 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "bf017ad89c558f9876d6474e10251b84b539065a", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "dadfa5b2ee9b3a7101a4463b1a4c9ca73f0c4b89", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "bf017ad89c558f9876d6474e10251b84b539065a", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index 538433175a..dfdaceb69f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -8,10 +8,10 @@ use polyglot_sql::traversal::ExpressionWalk; use serde_json::{Map, Value}; use crate::assembly::analysis_session::_helpers::dict_walk::{ - PyValue, alias_or_name, arg, children, column_table_name, direct_tables, expression_args, - find_all, is_star, kind, name, name_payload, output_name, parse_one, projected_expression, - py_truthy, python_int, raw_name_payload, set_operation, to_dict, top_level_ctes, - typed_annotation, unwrap_annotations, where_clause, + Assertion, PyValue, alias_or_name, arg, children, column_table_name, direct_tables, + expression_args, find_all, is_star, kind, name, name_payload, output_name, parse_one, + projected_expression, py_truthy, python_int, raw_name_payload, set_operation, to_dict, + top_level_ctes, typed_annotation, unwrap_annotations, where_clause, }; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::constants::{ @@ -705,9 +705,16 @@ impl Context<'_, '_> { { return Ok(Some(inferred)); } - let target: Value = - serde_json::to_value(asserted).map_err(|error| error.to_string())?; - return Ok(target.as_object().and_then(cast_target_type)); + return match asserted { + Assertion::Fixed(asserted) => { + let target: Value = + serde_json::to_value(asserted).map_err(|error| error.to_string())?; + Ok(target.as_object().and_then(cast_target_type)) + } + Assertion::Like(like) => Ok(self + .expression_type_in(Some(like), by_alias)? + .filter(|like| !upper(like).contains(UNKNOWN_TYPE))), + }; } let expression: Option<&Expression> = unwrap_annotations(expression); let node_kind: &str = kind(expression); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs index df9020df31..bbdf207c06 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs @@ -254,11 +254,22 @@ pub(crate) fn unwrap_annotations(mut node: Option<&Expression>) -> Option<&Expre node } +/// What a type assertion asserts: a type, or the type of another expression. +pub(crate) enum Assertion<'a> { + Fixed(&'a DataType), + Like(&'a Expression), +} + /// The first type assertion around `node`, past plain annotations, and the expression it types. -pub(crate) fn typed_annotation(mut node: Option<&Expression>) -> Option<(&DataType, &Expression)> { +pub(crate) fn typed_annotation( + mut node: Option<&Expression>, +) -> Option<(Assertion<'_>, &Expression)> { while let Some(Expression::Annotated(annotated)) = node { if let Some(asserted) = &annotated.type_assertion { - return Some((asserted, &annotated.this)); + return Some((Assertion::Fixed(asserted), &annotated.this)); + } + if let Some(like) = &annotated.type_like { + return Some((Assertion::Like(like), &annotated.this)); } node = Some(&annotated.this); } diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs index 7fdce06669..18a35b2a60 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs @@ -7,7 +7,10 @@ use crate::sql_macros::models::{ParameterKind, SqlMacroParameter}; pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String { let kind: String = match ¶meter.kind { ParameterKind::Expression { data_type } => format!("expression {}", type_text(data_type)), - ParameterKind::Any => "expression any".to_owned(), + ParameterKind::Any { family: None } => "expression any".to_owned(), + ParameterKind::Any { + family: Some(family), + } => format!("expression any {}", family.keyword().to_lowercase()), ParameterKind::Constant { data_type } => format!("const {}", type_text(data_type)), ParameterKind::Collection { set, data_type } => { format!("collection {set} {}", type_text(data_type)) diff --git a/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs index d06562d4dc..5c63caad55 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/main/sql_macro_definition_digest.rs @@ -12,11 +12,11 @@ pub fn sql_macro_definition_digest(name: &str, path: &str, source: &str) -> Stri }; let mut parts: Vec = vec![ definition.form.label().to_owned(), - definition - .returns - .as_deref() - .map(type_text) - .unwrap_or_default(), + match (&definition.returns, &definition.returns_like) { + (Some(returns), _) => type_text(returns), + (None, Some(like)) => format!("type of {}", like.to_lowercase()), + (None, None) => String::new(), + }, sql_definition_digest(&definition.body), ]; for parameter in &definition.parameters { diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index 73e7555a7c..3e3cbbdf0a 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -281,6 +281,7 @@ impl TypeContext<'_> { fn projection_type(&self, expression: &Expression, aliases: &Sources) -> Option { if asserted_expression(expression).is_none() + && unresolved_like(expression).is_none() && matches!(unwrapped(expression), Expression::Null(_)) { Some(NULL_TYPE.to_string()) @@ -296,6 +297,13 @@ impl TypeContext<'_> { } fn expression(&self, expression: &Expression, aliases: &Sources) -> Option { + if let Some((like, inner)) = unresolved_like(expression) { + let known = |data_type: &String| !data_type.to_ascii_uppercase().contains("UNKNOWN"); + return self + .expression(inner, aliases) + .filter(|inferred| known(inferred) && inferred != NULL_TYPE) + .or_else(|| self.expression(like, aliases).filter(known)); + } if let Some((asserted, inner)) = asserted_expression_type(expression) { if matches!(unwrapped(inner), Expression::Null(_)) { return self.cast_type(asserted); @@ -468,6 +476,24 @@ fn asserted_expression_type(mut expression: &Expression) -> Option<(&DataType, & } } +/// A typed-like assertion around `expression` that annotation has not resolved, past +/// parentheses and comments: the expression whose type it asserts, and what it types. +fn unresolved_like(mut expression: &Expression) -> Option<(&Expression, &Expression)> { + loop { + expression = match expression { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => { + return None; + } + Expression::Annotated(annotation) => match &annotation.type_like { + Some(like) => return Some((like, &annotation.this)), + None => &annotation.this, + }, + Expression::Paren(paren) => &paren.this, + _ => return None, + }; + } +} + fn unwrapped(mut expression: &Expression) -> &Expression { loop { expression = match expression { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs index 8a4d955e15..b799e90eec 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs @@ -351,11 +351,34 @@ pub(crate) fn check_shape( Err(expects("a SQL expression")) } (ParameterKind::Expression { .. }, _) => Ok(()), - (ParameterKind::Any, ArgumentShape::InlineCollection | ArgumentShape::Reference { .. }) => { + ( + ParameterKind::Any { .. }, + ArgumentShape::InlineCollection | ArgumentShape::Reference { .. }, + ) => { Err(expects("a scalar SQL expression") .helped("an ANY argument takes one value per row, such as a column, a literal or an expression; pass lists to LIST arguments and relations to RELATION arguments")) } - (ParameterKind::Any, _) => Ok(()), + ( + ParameterKind::Any { + family: Some(family), + }, + ArgumentShape::Literal, + ) => { + let actual: TypeFamily = literal_family(argument); + if family.admits(actual) { + Ok(()) + } else { + Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {name} of macro {macro_name}() expects {}; {shown} is {}", + family.noun(), + family_label(actual) + ), + )) + } + } + (ParameterKind::Any { .. }, _) => Ok(()), (ParameterKind::Constant { .. }, ArgumentShape::ConstantReference { .. }) => Ok(()), (ParameterKind::Constant { .. }, _) => Err(expects("a literal, a scalar constant or an enum value") .helped(format!("pass a literal, such as 2, or a constant, such as @const(\"{name}\"); a CONST argument is known at compile time, so a column or expression cannot be passed"))), @@ -392,10 +415,10 @@ pub(crate) fn check_shape( /// Whether an argument of kind `outer`, forwarded as `@name`, fits a parameter of kind `inner`. fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { match inner { - ParameterKind::Expression { .. } | ParameterKind::Any => matches!( + ParameterKind::Expression { .. } | ParameterKind::Any { .. } => matches!( outer, ParameterKind::Expression { .. } - | ParameterKind::Any + | ParameterKind::Any { .. } | ParameterKind::Constant { .. } | ParameterKind::Enum { .. } | ParameterKind::ColumnOf { .. } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs index fa16a29ae9..b894b6135f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs @@ -216,7 +216,9 @@ fn substituted( _ => argument.sql.clone(), } } - ParameterKind::Expression { .. } | ParameterKind::Any | ParameterKind::Constant { .. } + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::Constant { .. } if !is_atomic(&argument.sql) => { format!("({})", argument.sql) @@ -336,6 +338,7 @@ pub(crate) fn expand_body( sql: expanded[kept.0..kept.1].to_owned(), names, calls: within(calls, kept), + like: None, }) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs index 099a9940d6..da57a534db 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs @@ -15,7 +15,7 @@ use crate::sql_macros::constants::{ }; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ParameterKind, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, + ParameterKind, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, SqlMacroParameter, }; /// Locates a byte offset of the macro file. @@ -42,7 +42,17 @@ pub(crate) fn parsed_definition( let form: SqlMacroForm = body_form(body, &scanned); check_references(name, body, &scanned, &in_body)?; let calls: Vec = body_calls(name, body, &in_body)?; - check_returns(name, (form, header.returns.as_ref()), at(0))?; + check_returns( + name, + ( + form, + header.returns.is_some() || header.returns_like.is_some(), + ), + at(0), + )?; + if let Some((like, offset)) = &header.returns_like { + check_returns_like(name, (like, &header.parameters), at(*offset))?; + } let (body_line, body_column) = line_column(source, body_start); let definition: SqlMacroDefinition = SqlMacroDefinition { name: name.to_owned(), @@ -50,6 +60,7 @@ pub(crate) fn parsed_definition( description: header.description, parameters: header.parameters, returns: header.returns, + returns_like: header.returns_like.map(|(like, _)| like), form, body: body.to_owned(), body_line, @@ -157,11 +168,11 @@ fn body_calls(name: &str, body: &str, in_body: Locate<'_>) -> Result /// Rejects `returns` on a CTE-list or column-list macro. fn check_returns( name: &str, - (form, returns): (SqlMacroForm, Option<&String>), + (form, returns): (SqlMacroForm, bool), location: SqlMacroLocation, ) -> Result<(), SqlMacroError> { match (form, returns) { - (SqlMacroForm::CteList | SqlMacroForm::ColumnList, Some(_)) => Err(SqlMacroError::new( + (SqlMacroForm::CteList | SqlMacroForm::ColumnList, true) => Err(SqlMacroError::new( INVALID_DEFINITION_CODE, format!( "{} macro {name}() cannot declare returns; only expression macros return a value", @@ -173,6 +184,50 @@ fn check_returns( } } +/// Rejects `returns TYPE OF` naming no argument, or an argument with no scalar type. +fn check_returns_like( + name: &str, + (like, parameters): (&str, &[SqlMacroParameter]), + location: SqlMacroLocation, +) -> Result<(), SqlMacroError> { + let Some(parameter) = parameters + .iter() + .find(|parameter| parameter.name.eq_ignore_ascii_case(like)) + else { + let names: Vec<&str> = parameters + .iter() + .map(|parameter| parameter.name.as_str()) + .collect(); + return Err(SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!("returns TYPE OF {like} names no argument of macro {name}()"), + ) + .at(location) + .helped(if names.is_empty() { + format!("macro {name}() has no arguments; declare returns instead") + } else { + format!("name one of its arguments: {}", names.join(", ")) + })); + }; + match parameter.kind { + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::Constant { .. } + | ParameterKind::Enum { .. } => Ok(()), + ParameterKind::Collection { .. } + | ParameterKind::Relation { .. } + | ParameterKind::ColumnOf { .. } => Err(SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!( + "returns TYPE OF {} names a {} argument of macro {name}(); only a scalar argument has a type to return", + parameter.name, + parameter.kind.label() + ), + ) + .at(location)), + } +} + /// Rejects an `@name` naming no parameter and a RELATION parameter written bare. fn check_parameter_references( definition: &SqlMacroDefinition, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index 0900cc3aa1..a029bdd94d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -16,7 +16,7 @@ use crate::sql_macros::_helpers::literal_positions::{ use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; use crate::sql_macros::_helpers::placement::top_level_parts; use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; -use crate::sql_macros::_helpers::returns_check::check_body_returns; +use crate::sql_macros::_helpers::returns_check::{check_body_returns, check_body_types}; use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ ARGUMENT_CODE, CTE_ARGUMENT, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, @@ -240,8 +240,23 @@ pub(crate) fn expand_call( &body, (host.adapter(), body_location(&definition)), )?; - let expansion: BodyExpansion = expand_body(&definition, &body, &rendered) + check_body_types( + &definition, + &body, + (host.adapter(), body_location(&definition)), + )?; + let mut expansion: BodyExpansion = expand_body(&definition, &body, &rendered) .map_err(|error| error.at(body_location(&definition)))?; + expansion.like = definition.returns_like.as_deref().and_then(|name| { + rendered + .iter() + .find(|argument| { + definition.parameters[argument.parameter] + .name + .eq_ignore_ascii_case(name) + }) + .map(|argument| argument.sql.clone()) + }); if call.in_python { return Ok(( ExpandedText { @@ -336,7 +351,7 @@ fn rendered_argument( } } ( - ParameterKind::Constant { .. } | ParameterKind::Any, + ParameterKind::Constant { .. } | ParameterKind::Any { .. }, ArgumentShape::ConstantReference { name }, ) => { if host.constant_shape(scope, name)? != ConstantShape::Scalar { @@ -408,6 +423,7 @@ fn placed( end: text.len() - comma, macro_name: definition.name.clone(), returns: definition.returns.clone(), + returns_like: expansion.like.clone(), }); } Ok((ExpandedText { sql: text, calls }, inserted)) @@ -607,9 +623,9 @@ fn check_adapter_types( .iter() .map(|(_, data_type)| data_type.as_str()) .collect(), - ParameterKind::Any | ParameterKind::Enum { .. } | ParameterKind::ColumnOf { .. } => { - Vec::new() - } + ParameterKind::Any { .. } + | ParameterKind::Enum { .. } + | ParameterKind::ColumnOf { .. } => Vec::new(), }; for data_type in types { if let Some(error) = invalid( @@ -659,6 +675,14 @@ fn argument_checks( data_type: data_type.clone(), } } + ( + ParameterKind::Any { + family: Some(family), + }, + ArgumentShape::Expression | ArgumentShape::Identifier { .. }, + ) if !rendered.sql.contains('@') && !rendered.sql.contains("__") => { + ArgumentExpectation::Family { family: *family } + } _ => continue, }; checks.push(ArgumentCheck { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs index e644af2802..2704b94e98 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs @@ -5,11 +5,11 @@ use crate::sql_macros::constants::{ ANY_KEYWORD, ARGUMENTS_KEY, COLUMN_KEYWORD, COMMA, CONST_KEYWORD, DEFAULT_KEYWORD, DESCRIPTION_KEY, ENUM_KEYWORD, HEADER_KEYWORD, INVALID_DEFINITION_CODE, LIST_KEYWORD, OF_KEYWORD, RAW_TYPE_NAMES, RELATION_COLUMN_MIN_TOKENS, RELATION_KEYWORD, RETURNS_KEY, - SET_KEYWORD, + SET_KEYWORD, TYPE_KEYWORD, }; use crate::sql_macros::errors::LexError; use crate::sql_macros::errors::SqlMacroError; -use crate::sql_macros::models::{ParameterKind, SqlMacroLocation, SqlMacroParameter}; +use crate::sql_macros::models::{AnyFamily, ParameterKind, SqlMacroLocation, SqlMacroParameter}; use sqlbuild_sqltext::header_strings::constants::{ DOUBLE_QUOTE_INSIDE_HELP, DOUBLE_QUOTE_INSIDE_MESSAGE, SINGLE_QUOTE_INSIDE_HELP, SINGLE_QUOTE_INSIDE_MESSAGE, TRIPLE_QUOTE_INSIDE_HELP, TRIPLE_QUOTE_INSIDE_MESSAGE, @@ -22,6 +22,8 @@ pub(crate) struct Header { pub(crate) description: Option, pub(crate) parameters: Vec, pub(crate) returns: Option, + /// The argument named by `returns TYPE OF `, and its byte offset. + pub(crate) returns_like: Option<(String, usize)>, /// Byte offset just past the header and its optional `;`. pub(crate) body_start: usize, } @@ -83,6 +85,7 @@ impl HeaderParser<'_> { description: None, parameters: Vec::new(), returns: None, + returns_like: None, body_start, }; for entry in split_top_level(source, &scanned[2..close]) { @@ -145,6 +148,19 @@ impl HeaderParser<'_> { if value.is_empty() { return Err(self.error(key.start, "returns needs a SQL type")); } + if let [type_word, of, argument] = value + && type_word.is_word(source, TYPE_KEYWORD) + && of.is_word(source, OF_KEYWORD) + { + if argument.kind != TokenKind::Word { + return Err(self.error( + argument.start, + "returns TYPE OF needs an argument name, such as returns TYPE OF value", + )); + } + header.returns_like = Some((argument.text(source).to_owned(), argument.start)); + return Ok(header); + } header.returns = Some(type_text(source, value)); Ok(header) } @@ -288,22 +304,38 @@ impl HeaderParser<'_> { } if first.is_word(source, ANY_KEYWORD) { let Some(after) = tokens.get(1) else { - return Ok(ParameterKind::Any); + return Ok(ParameterKind::Any { family: None }); }; + let family: Option = AnyFamily::ALL + .into_iter() + .find(|family| after.is_word(source, family.keyword())); + if let Some(family) = family { + let Some(extra) = tokens.get(2) else { + return Ok(ParameterKind::Any { + family: Some(family), + }); + }; + return Err(self + .error(extra.start, format!("argument {name} ANY {} takes nothing after it; CONST, LIST and SET are not supported for ANY families", family.keyword())) + .helped(format!("declare it {name} ANY {}, or give it a SQL type", family.keyword()))); + } if after.is_word(source, CONST_KEYWORD) { return Err(self .error(after.start, format!("argument {name} ANY CONST: a CONST argument needs a type its values are checked against")) .helped(format!("declare it with the type of its values, such as {name} VARCHAR CONST"))); } return Err(self - .error(after.start, format!("argument {name} ANY takes nothing after it; ANY accepts any scalar expression")) - .helped(format!("declare it {name} ANY, or give it a SQL type"))); + .error(after.start, format!("argument {name} ANY takes nothing after it but NUMERIC, TEXT or TEMPORAL; ANY accepts any scalar expression")) + .helped(format!("declare it {name} ANY, or {name} ANY NUMERIC, ANY TEXT or ANY TEMPORAL for one type family, or give it a SQL type"))); } if first.is_word(source, LIST_KEYWORD) || first.is_word(source, SET_KEYWORD) { let set: bool = first.is_word(source, SET_KEYWORD); let keyword: String = first.text(source).to_ascii_uppercase(); let element: &[Token] = &tokens[1..]; - if element.len() == 1 && element[0].is_word(source, ANY_KEYWORD) { + if element + .first() + .is_some_and(|first| first.is_word(source, ANY_KEYWORD)) + { return Err(self .error( element[0].start, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs index 515eb037e5..6d24941fdc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs @@ -39,7 +39,9 @@ pub(crate) fn non_literal_use<'r>( let parameter = definition.parameter(&reference.name)?; if !matches!( parameter.kind, - ParameterKind::Expression { .. } | ParameterKind::Any | ParameterKind::ColumnOf { .. } + ParameterKind::Expression { .. } + | ParameterKind::Any { .. } + | ParameterKind::ColumnOf { .. } ) { return None; } @@ -232,7 +234,7 @@ pub(crate) fn literal_error( Some(ParameterKind::Expression { data_type }) => { format!("declare it {name} {data_type} CONST") } - Some(ParameterKind::Any) => { + Some(ParameterKind::Any { .. }) => { format!("declare it with a type and CONST, such as {name} VARCHAR CONST") } _ => format!("pass the value through a CONST argument, such as {name}_value VARCHAR CONST"), diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs index 472251f765..3740d03294 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs @@ -193,7 +193,7 @@ pub(crate) fn probe_plan(sql: &str, checks: &[SqlMacroCheck]) -> ProbePlan { evidence[index] = Some(Evidence::Columns(plan.probes.len() - 1)); } } - ArgumentExpectation::Typed { .. } => { + ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let group: usize = match typed_probe_index(&scopes, &check.scope) { Some(group) => group, None => { @@ -345,7 +345,7 @@ pub(crate) fn check_findings( )); } } - ArgumentExpectation::Typed { data_type } => { + ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let Some(Some(Evidence::Column(probe, column))) = evidence.get(index) else { continue; }; @@ -355,6 +355,28 @@ pub(crate) fn check_findings( let Some(Some(actual)) = column_type(found, column) else { continue; }; + let data_type: &str = match &argument.expected { + ArgumentExpectation::Family { family } => { + if !family.admits(type_family(actual, dialect)) { + findings.push(finding( + argument, + format!( + "argument {parameter} of macro {name}() expects {}; {} is {}", + family.noun(), + argument.text, + shown_type(actual) + ), + Some(format!( + "pass {} or declare {parameter} with a wider type", + family.noun() + )), + )); + } + continue; + } + ArgumentExpectation::Typed { data_type } => data_type, + _ => continue, + }; if !compatible(Some(actual), data_type, dialect) { findings.push(finding( argument, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs index 8069d96f4d..b22da7d5c0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; -use polyglot_sql::ValidationSchema; +use polyglot_sql::{DialectType, SchemaValidationOptions, ValidationSchema, validate_with_schema}; use serde_json::{Value, json}; use crate::contracts::_helpers::type_comparison::{TypeComparer, TypeComparison}; @@ -12,7 +12,8 @@ use crate::sql_macros::_helpers::arguments::type_family; use crate::sql_macros::_helpers::body::expand_body; use crate::sql_macros::_helpers::typed_calls::typed_sql; use crate::sql_macros::constants::{ - INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, + BODY_TYPE_CODE, INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, TYPE_ERROR_CODES, + TYPE_WARNING_CODES, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, }; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ @@ -32,14 +33,18 @@ pub(crate) fn adapter_dialect(adapter: &str) -> &str { } } -/// The type a parameter's stand-in column has, or `None` for an untyped stand-in. -fn stand_in_type(kind: &ParameterKind) -> Option<&str> { +/// The type a parameter's stand-in column has, or `None` for an untyped stand-in; an `ANY` +/// family takes its widest member when `families` is set. +fn stand_in_type(kind: &ParameterKind, families: bool) -> Option<&str> { match kind { ParameterKind::Expression { data_type } | ParameterKind::Constant { data_type } => { Some(data_type) } ParameterKind::Collection { data_type, .. } => Some(data_type), - ParameterKind::Any + ParameterKind::Any { + family: Some(family), + } if families => Some(family.widest()), + ParameterKind::Any { .. } | ParameterKind::Enum { .. } | ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. } => None, @@ -51,8 +56,13 @@ fn stand_in_column(index: usize) -> String { format!("p{index}") } -/// The body's type with typed stand-ins for its arguments, unless it reads relations or columns. -fn body_probe(definition: &SqlMacroDefinition, body: &ExpandedText) -> Option<(String, Value)> { +/// The query selecting the body with stand-ins for its arguments, and the stand-ins' relation, +/// unless the body reads relations or columns. +fn body_probe( + definition: &SqlMacroDefinition, + body: &ExpandedText, + families: bool, +) -> Option<(String, Value)> { let mut arguments: Vec = Vec::with_capacity(definition.parameters.len()); let mut columns: Vec = Vec::with_capacity(definition.parameters.len()); for (index, parameter) in definition.parameters.iter().enumerate() { @@ -69,7 +79,7 @@ fn body_probe(definition: &SqlMacroDefinition, body: &ExpandedText) -> Option<(S }; columns.push(json!({ "name": column, - "type": stand_in_type(¶meter.kind).unwrap_or(UNKNOWN_TYPE_HINT), + "type": stand_in_type(¶meter.kind, families).unwrap_or(UNKNOWN_TYPE_HINT), })); arguments.push(RenderedArgument { parameter: index, @@ -122,11 +132,11 @@ pub(crate) fn check_body_returns( let Some(returns) = definition.returns.as_deref() else { return Ok(()); }; - let Some((probe, table)) = body_probe(definition, body) else { + let Some((probe, table)) = body_probe(definition, body, false) else { return Ok(()); }; let dialect: &str = adapter_dialect(adapter); - let key: String = format!("{dialect}\n{table}\n{probe}"); + let key: String = format!("returns\n{dialect}\n{table}\n{probe}"); let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); let known: Option> = verdicts .lock() @@ -165,3 +175,86 @@ pub(crate) fn check_body_returns( "declare returns {shown}, CAST the body to {returns}, or remove returns to use the inferred type" ))) } + +/// The first type error the body's probe shows, as polyglot reports it. +fn body_type_error(probe: &str, table: Value, dialect: &str) -> Option { + let schema: ValidationSchema = serde_json::from_value(json!({ "tables": [table] })).ok()?; + let dialect_type: DialectType = dialect.parse().ok()?; + let options: SchemaValidationOptions = SchemaValidationOptions { + check_types: true, + strict: Some(false), + ..Default::default() + }; + validate_with_schema(probe, dialect_type, &schema, &options) + .errors + .into_iter() + .find(|error| is_type_code(&error.code)) + .map(|error| error.message) +} + +/// Whether a polyglot validation code reports an operand or argument of the wrong type. +fn is_type_code(code: &str) -> bool { + let Some(number) = code + .get(1..) + .and_then(|digits| digits.parse::().ok()) + else { + return false; + }; + (code.starts_with('E') && (TYPE_ERROR_CODES.0..=TYPE_ERROR_CODES.1).contains(&number)) + || (code.starts_with('W') + && (TYPE_WARNING_CODES.0..=TYPE_WARNING_CODES.1).contains(&number)) +} + +/// Rejects a body that misuses an `ANY` family argument, typed as the family's widest member. +pub(crate) fn check_body_types( + definition: &SqlMacroDefinition, + body: &ExpandedText, + (adapter, location): (&str, SqlMacroLocation), +) -> Result<(), SqlMacroError> { + let families: Vec = definition + .parameters + .iter() + .filter_map(|parameter| match parameter.kind { + ParameterKind::Any { + family: Some(family), + } => Some(format!("{} ANY {}", parameter.name, family.keyword())), + _ => None, + }) + .collect(); + if families.is_empty() { + return Ok(()); + } + let Some((probe, table)) = body_probe(definition, body, true) else { + return Ok(()); + }; + let dialect: &str = adapter_dialect(adapter); + let key: String = format!("types\n{dialect}\n{table}\n{probe}"); + let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); + let known: Option> = verdicts + .lock() + .ok() + .and_then(|verdicts| verdicts.get(&key).cloned()); + let found: Option = match known { + Some(found) => found, + None => { + let found: Option = body_type_error(&probe, table, dialect); + if let Ok(mut verdicts) = verdicts.lock() { + verdicts.insert(key, found.clone()); + } + found + } + }; + let Some(found) = found else { + return Ok(()); + }; + Err(SqlMacroError::new( + BODY_TYPE_CODE, + format!( + "body of macro {}() does not accept every value of {}: {found}", + definition.name, + families.join(", ") + ), + ) + .at(location) + .helped("an ANY family argument may be any type of its family; the body is checked with the family's widest type, so CAST the argument or narrow its declared type")) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs index a141eb8446..dee823347a 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs @@ -1,6 +1,6 @@ //! Expression macro calls followed through the edits that move them, and their typed markers. -use polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION; +use polyglot_sql::expressions::{TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION}; use crate::compiled_project::models::SplicedSpan; use crate::sql_macros::_helpers::body::with_line_comment_closed; @@ -97,8 +97,13 @@ fn quoted(returns: &str) -> String { pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { let mut edits: Vec = Vec::new(); for (index, call) in calls.iter().enumerate() { - let Some(returns) = &call.returns else { - continue; + let (function, asserted): (&str, String) = match (&call.returns, &call.returns_like) { + (Some(returns), _) => (TYPED_EXPRESSION_FUNCTION, quoted(returns)), + (None, Some(like)) => ( + TYPED_LIKE_EXPRESSION_FUNCTION, + with_line_comment_closed(like, true), + ), + (None, None) => continue, }; let valid = |position: usize| position <= sql.len() && sql.is_char_boundary(position); if call.start >= call.end || !valid(call.start) || !valid(call.end) { @@ -114,13 +119,13 @@ pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec Vec<(String, String)> { - let untyped: Vec<&TypedCall> = calls.iter().filter(|call| call.returns.is_none()).collect(); + let untyped: Vec<&TypedCall> = calls + .iter() + .filter(|call| call.returns.is_none() && call.returns_like.is_none()) + .collect(); if untyped.is_empty() { return Vec::new(); } diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index c2912df89f..83f818406d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -29,8 +29,8 @@ impl SqlMacroForm { pub enum ParameterKind { /// Any SQL expression of a compatible type. Expression { data_type: String }, - /// Any scalar SQL expression, whatever its type. - Any, + /// Any scalar SQL expression, whatever its type, or any of one type family. + Any { family: Option }, /// A literal or a scalar constant. Constant { data_type: String }, /// An inline list or set, or a list or set constant; `set` for `SET`. @@ -48,7 +48,10 @@ impl ParameterKind { pub fn label(&self) -> String { match self { Self::Expression { data_type } => data_type.clone(), - Self::Any => "ANY".to_owned(), + Self::Any { family: None } => "ANY".to_owned(), + Self::Any { + family: Some(family), + } => format!("ANY {}", family.keyword()), Self::Constant { data_type } => format!("{data_type} CONST"), Self::Collection { set, data_type } => { format!("{} {data_type}", if *set { "SET" } else { "LIST" }) @@ -66,6 +69,66 @@ impl ParameterKind { } } +/// A type family an `ANY` parameter may be narrowed to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnyFamily { + /// Integer, decimal and float types. + Numeric, + /// String types. + Text, + /// Date, time and timestamp types. + Temporal, +} + +impl AnyFamily { + /// Every family, in the order messages list them. + pub const ALL: [Self; 3] = [Self::Numeric, Self::Text, Self::Temporal]; + + /// The word after `ANY` that declares the family. + pub fn keyword(self) -> &'static str { + match self { + Self::Numeric => "NUMERIC", + Self::Text => "TEXT", + Self::Temporal => "TEMPORAL", + } + } + + /// The family's widest member, which a macro body is type-checked against. + pub fn widest(self) -> &'static str { + match self { + Self::Numeric => "DOUBLE", + Self::Text => "VARCHAR", + Self::Temporal => "TIMESTAMP", + } + } + + /// What the family's values are, as messages name them. + pub fn noun(self) -> &'static str { + match self { + Self::Numeric => "a number", + Self::Text => "text", + Self::Temporal => "a date, time or timestamp", + } + } + + /// Whether a value of `family` belongs to this family; unknown and NULL values do. + pub(crate) fn admits(self, family: TypeFamily) -> bool { + matches!( + (self, family), + (_, TypeFamily::Unknown | TypeFamily::Null) + | ( + Self::Numeric, + TypeFamily::Integer | TypeFamily::Decimal | TypeFamily::Float + ) + | (Self::Text, TypeFamily::Text) + | ( + Self::Temporal, + TypeFamily::Date | TypeFamily::Timestamp | TypeFamily::Time + ) + ) + } +} + /// One declared parameter. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SqlMacroParameter { @@ -88,6 +151,8 @@ pub struct SqlMacroDefinition { pub parameters: Vec, /// The declared result type of an expression macro. pub returns: Option, + /// The parameter whose argument's type an expression macro returns, by `returns TYPE OF`. + pub returns_like: Option, pub form: SqlMacroForm, /// The body as written, after the header. pub body: String, @@ -177,6 +242,8 @@ pub struct BodyExpansion { pub names: Vec, /// The expression macro calls the SQL holds, from the body and the arguments. pub calls: Vec, + /// The SQL of the argument whose type an expression macro call returns, by `returns TYPE OF`. + pub like: Option, } /// One expression macro call's expansion in a text: its byte range and the macro's `returns`. @@ -187,6 +254,9 @@ pub struct TypedCall { pub macro_name: String, /// The macro's declared result type; `None` when it declares none. pub returns: Option, + /// The SQL of the argument whose type the call returns, by `returns TYPE OF`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub returns_like: Option, } /// SQL an expansion produced and the expression macro calls it holds. @@ -326,6 +396,8 @@ pub enum ArgumentExpectation { ColumnOf { relation: String }, /// An expression of a type compatible with this one. Typed { data_type: String }, + /// An expression of a type in this family. + Family { family: AnyFamily }, } /// A call argument the caller's analysed SQL shows does not fit its parameter. diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs index b25a71ef65..fc7b05b3df 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs @@ -196,6 +196,36 @@ fn given_invalid_macro_files_when_parsing_then_each_is_rejected() { expected_code: "P015", expected_message: "argument n ANY takes nothing after it", }, + DefinitionErrorTestCase { + description: "ANY family with CONST", + source: "MACRO (description \"x\", arguments (n ANY NUMERIC CONST), returns BIGINT);\n@n", + expected_code: "P015", + expected_message: "argument n ANY NUMERIC takes nothing after it", + }, + DefinitionErrorTestCase { + description: "LIST of an ANY family", + source: "MACRO (description \"x\", arguments (codes LIST ANY TEXT), returns BOOLEAN);\n'a' IN @codes", + expected_code: "P015", + expected_message: "argument codes LIST ANY: ANY is for scalar arguments only", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF an unknown argument", + source: "MACRO (description \"x\", arguments (v ANY), returns TYPE OF w);\n@v", + expected_code: "P015", + expected_message: "returns TYPE OF w names no argument of macro m()", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF a LIST argument", + source: "MACRO (description \"x\", arguments (v LIST VARCHAR), returns TYPE OF v);\n'a' IN @v", + expected_code: "P015", + expected_message: "returns TYPE OF v names a LIST VARCHAR argument of macro m(); only a scalar argument has a type to return", + }, + DefinitionErrorTestCase { + description: "returns TYPE OF on a CTE list", + source: "MACRO (description \"x\", arguments (src RELATION (id BIGINT)), returns TYPE OF src);\nWITH a AS (SELECT * FROM @src)", + expected_code: "P015", + expected_message: "CTE-list macro m() cannot declare returns", + }, DefinitionErrorTestCase { description: "LIST of ANY", source: "MACRO (description \"x\", arguments (codes LIST ANY), returns BOOLEAN);\n'a' IN @codes", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index d057c1d072..c15ba68538 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -137,6 +137,14 @@ const SHOUTED_NESTED: (&str, &str) = ( "shouted_twice", "MACRO (description \"Shouted\", arguments (value VARCHAR), returns BIGINT);\n\n@sql.plus_one(LENGTH(@value)) || 'x'\n", ); +const ROUND_ANY: (&str, &str) = ( + "round_any", + "MACRO (description \"Rounded\", arguments (v ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF v);\n\nROUND(@v, @places)\n", +); +const SHOUT_NUMBER: (&str, &str) = ( + "shout_number", + "MACRO (description \"Shouted\", arguments (v ANY NUMERIC));\n\nUPPER(@v)\n", +); const ODD: (&str, &str) = ( "odd", "MACRO (description \"Odd\", arguments (v FOO BAR), returns BIGINT);\n\n@v + 1\n", @@ -425,6 +433,13 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "SELECT @sql.odd(1) AS a", expected_sql: "SELECT 1 + 1 AS a", }, + ExpansionTestCase { + description: "an ANY family argument of its family", + adapter: "duckdb", + macros: &[ROUND_ANY], + sql: "SELECT @sql.round_any(o.amount, 1) AS a FROM orders AS o", + expected_sql: "SELECT ROUND(o.amount, 1) AS a FROM orders AS o", + }, ExpansionTestCase { description: "an expression macro without returns", adapter: "duckdb", @@ -528,6 +543,22 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { expected_code: "P024", expected_message: "macro shouted_twice() declares returns BIGINT, but its body is VARCHAR", }, + ExpansionErrorTestCase { + description: "a literal outside an ANY family", + adapter: "duckdb", + macros: &[ROUND_ANY], + sql: "SELECT @sql.round_any('ten')", + expected_code: "P017", + expected_message: "argument v of macro round_any() expects a number; 'ten' is a string", + }, + ExpansionErrorTestCase { + description: "a body that misuses an ANY family argument", + adapter: "duckdb", + macros: &[SHOUT_NUMBER], + sql: "SELECT @sql.shout_number(o.amount) FROM orders AS o", + expected_code: "P025", + expected_message: "body of macro shout_number() does not accept every value of v ANY NUMERIC", + }, ExpansionErrorTestCase { description: "cycle", adapter: "", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index e847311788..98e15a0ad0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -1,6 +1,6 @@ use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use crate::sql_macros::main::untyped_call_outputs::untyped_call_outputs; -use crate::sql_macros::models::TypedCall; +use crate::sql_macros::models::{ExpandedText, TypedCall}; use crate::sql_macros::tests::helpers::expanded_text_for; use crate::sql_macros::tests::test_types::{TypedAnalysisTestCase, TypedCallTestCase}; @@ -166,6 +166,7 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns .calls .iter() .map(|(start, end, returns)| TypedCall { + returns_like: None, start: *start, end: *end, macro_name: "m".to_owned(), @@ -199,6 +200,7 @@ fn given_the_reserved_analysis_function_when_comparing_then_authored_sql_rejects #[test] fn given_calls_without_returns_when_listing_outputs_then_only_whole_named_items_are_named() { let untyped = |start: usize, end: usize| TypedCall { + returns_like: None, start, end, macro_name: "shout".to_owned(), @@ -224,3 +226,27 @@ fn given_calls_without_returns_when_listing_outputs_then_only_whole_named_items_ ] ); } + +const LIKE_VALUE: (&str, &str) = ( + "like_value", + "MACRO (description \"Like\", arguments (value ANY, fallback ANY DEFAULT NULL), \ + returns TYPE OF value);\n\nCOALESCE(@value, @fallback)\n", +); + +#[test] +fn given_a_returns_type_of_call_when_building_the_analysis_text_then_it_asserts_its_argument() { + let expanded: ExpandedText = expanded_text_for( + "duckdb", + &[LIKE_VALUE], + "SELECT @sql.like_value(o.created_at + 1) AS v FROM orders AS o", + ) + .expect("expands"); + + let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls); + + assert_eq!( + typed, + "SELECT __polyglot_typed_like(COALESCE((o.created_at + 1), NULL), o.created_at + 1) \ + AS v FROM orders AS o" + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index 168775c6d6..b06802b89f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::main::text_position::text_position; use sqlbuild_analysis::sql_macros::constants::{ - BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, MIXED_KIND_CODE, RETURNS_MISMATCH_CODE, + BARE_MACRO_CALL_CODE, BODY_SCOPE_CODE, BODY_TYPE_CODE, MIXED_KIND_CODE, RETURNS_MISMATCH_CODE, }; use sqlbuild_analysis::sql_macros::errors::ExpansionFailure; use sqlbuild_analysis::sql_macros::errors::SqlMacroError; @@ -149,8 +149,12 @@ pub(crate) fn expanded_sql_macro_calls<'py>( Err(ExpansionFailure::Host(error)) => return Err(error), Err(ExpansionFailure::Macro(error)) => { let located: SqlMacroError = match host.call_location((call.start, call.end)) { - // A wrong returns is the macro file's error; the call comes after it. - Some(location) if error.code == RETURNS_MISMATCH_CODE => error.at(location), + // A body that disagrees with its types is the macro file's error. + Some(location) + if [RETURNS_MISMATCH_CODE, BODY_TYPE_CODE].contains(&error.code) => + { + error.at(location) + } Some(location) => error.at_call(location), None => error, }; diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs index edb42e2cb1..f217c67301 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs @@ -192,7 +192,9 @@ impl OutputRule<'_> { .flatten(); let passes = cast_matches_contract || column.type_proven && direct && !scope.set_branch - || column.type_proven && call.is_some_and(|call| call.returns.is_some()); + || column.type_proven + && call + .is_some_and(|call| call.returns.is_some() || call.returns_like.is_some()); if passes { continue; } @@ -212,7 +214,7 @@ impl OutputRule<'_> { "output {:?} {reason}; declared type is {}", name, column.data_type ), - match call.filter(|call| call.returns.is_none()) { + match call.filter(|call| call.returns.is_none() && call.returns_like.is_none()) { Some(call) => format!( "Declare returns {} in SQL macro {}(), or wrap the call in CAST(... AS {}).", column.data_type, call.macro_name, column.data_type diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index ad85b6351a..9f5149616b 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -797,6 +797,15 @@ def _sql_macro_cases() -> tuple[FailureCase, ...]: ), ), ), + failure_case( + name="sql-macro-body-misuses-an-any-family", + expected_code="P025", + expected_message="body of macro x() does not accept every value of v ANY NUMERIC", + files=_sql_macro_columns( + columns="@sql.x(amount) AS amount, status", + files=_x_macro(arguments="v ANY NUMERIC", returns="", body="UPPER(@v)\n"), + ), + ), failure_case( name="sql-macro-body-declaration-not-visible", expected_code="S026", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 288116e46c..94ccae6a9c 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -565,6 +565,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "P022", "P023", "P024", + "P025", "S026", } ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index 6600e400cd..ca01b160e4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -2,6 +2,7 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest @@ -11,11 +12,13 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( contract_model, + orders_model, output, run_project, text_macro, with_rules, ) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb _UNTYPED_UPSTREAM: dict[str, str] = { "models/marts/up.sql": ( @@ -140,3 +143,324 @@ def test_given_optional_returns_when_compiling_then_calls_are_typed_and_checked( assert fragment in output(result), output(result) for fragment in test_case.unexpected_fragments: assert fragment not in output(result), output(result) + + +_FIRST_OF: dict[str, str] = { + "macros/sql/first_of.sql": ( + 'MACRO (\n description "The first of two values",\n' + " arguments (value ANY, other ANY),\n returns TYPE OF value,\n);\n\n" + "list_extract([@value, @other], 1)\n" + ) +} +_FIRST_OF_UNTYPED: dict[str, str] = { + "macros/sql/first_of.sql": _FIRST_OF["macros/sql/first_of.sql"].replace( + " returns TYPE OF value,\n", "" + ) +} +_DAY: str = "CAST(o.{column} AS DATE)" +_DAYS: str = f"{_DAY.format(column='created_at')}, {_DAY.format(column='updated_at')}" +_TYPE_OF_POSITIONS: str = ( + "WITH c AS (\n" + f" SELECT o.order_id, @sql.first_of({_DAYS}) AS in_cte\n" + ' FROM __ref("stg_orders") AS o\n' + ")\n" + "SELECT\n" + " @sql.first_of(o.status, o.country_code) AS text_value,\n" + f" @sql.first_of({_DAYS}) AS day_value,\n" + " c.in_cte,\n" + f" COALESCE(@sql.first_of({_DAYS}), {_DAY.format(column='created_at')}) AS nested\n" + 'FROM __ref("stg_orders") AS o\nJOIN c ON c.order_id = o.order_id\n' +) +_TYPE_OF_COLUMNS: str = ( + "text_value (type VARCHAR), day_value (type DATE), in_cte (type DATE), nested (type DATE)" +) +_TYPE_OF_UNION: str = ( + 'SELECT CAST(NULL AS DATE) AS day_value FROM __ref("stg_orders") AS o\n' + "UNION ALL\n" + f'SELECT @sql.first_of({_DAYS}) AS day_value FROM __ref("stg_orders") AS o\n' +) +_VARIANT_PATHS: str = ( + "WITH raw AS (SELECT PARSE_JSON('{{\"a\": 1}}') AS payload)\n" + "SELECT @sql.payload_value({path}) AS v FROM raw\n" +) +_SNOWFLAKE_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" + ) +} + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="returns_type_of_gives_each_call_its_argument_type", + files={ + **_FIRST_OF, + **contract_model("orders", _TYPE_OF_COLUMNS, _TYPE_OF_POSITIONS), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_types_a_later_union_branch", + files={ + **with_rules("SQBRCONTRACT105"), + **_FIRST_OF, + **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="without_returns_the_same_calls_stay_untyped", + files={ + **_FIRST_OF_UNTYPED, + **contract_model("orders", _TYPE_OF_COLUMNS, _TYPE_OF_POSITIONS), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'text_value' type could not be proven against declared " + "VARCHAR", + "warning[K003]: column 'day_value' type could not be proven against declared DATE", + "warning[K003]: column 'in_cte' type could not be proven against declared DATE", + "declare returns DATE in SQL macro first_of()", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_cast_variant_path_is_variant", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("payload_value", "TYPE OF value", "@value", argument="ANY"), + **contract_model( + "orders", + "v (type VARIANT)", + _VARIANT_PATHS.format(path="raw.payload:a::VARIANT"), + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_bare_variant_path_stays_unknown_like_the_path", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("payload_value", "TYPE OF value", "@value", argument="ANY"), + **contract_model( + "orders", "v (type VARIANT)", _VARIANT_PATHS.format(path="raw.payload:a") + ), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared VARIANT", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_an_unknown_argument_stays_unknown", + files={ + **_FIRST_OF, + **contract_model( + "orders", + "v (type VARCHAR)", + "SELECT @sql.first_of(bar(o.amount_cents, 0, 10), o.status) AS v\n" + 'FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=( + "warning[K003]: column 'v' type could not be proven against declared VARCHAR", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_keeps_a_known_body_type_that_differs", + files={ + **text_macro("m", "TYPE OF value", "LENGTH(@value)", argument="ANY"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'v' inferred as BIGINT",), + ), + SqlMacroCompileTestCase( + description="returns_type_of_an_unknown_argument_name_is_reported_at_the_macro_file", + files={ + **text_macro("m", "TYPE OF missing", "@value", argument="ANY"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns TYPE OF missing names no argument of macro m()", + "macros/sql/m.sql:4:", + ), + ), + SqlMacroCompileTestCase( + description="returns_type_of_a_list_argument_is_reported_at_the_macro_file", + files={ + "macros/sql/m.sql": ( + 'MACRO (\n description "Listed",\n arguments (codes LIST VARCHAR),\n' + " returns TYPE OF codes,\n);\n\n'GB' IN @codes\n" + ), + **contract_model( + "orders", + "v (type BOOLEAN)", + "SELECT @sql.m(['GB']) AS v FROM __ref(\"stg_orders\") AS o\n", + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns TYPE OF codes names a LIST VARCHAR argument of macro m()", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_returns_type_of_when_compiling_then_each_call_has_its_argument_type( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +_ROUNDED: dict[str, str] = { + "macros/sql/rounded.sql": ( + 'MACRO (\n description "Rounded to places",\n' + " arguments (value ANY NUMERIC, places INT CONST DEFAULT 2),\n" + " returns TYPE OF value,\n);\n\nROUND(@value, @places)\n" + ), + "macros/sql/shouted.sql": ( + 'MACRO (\n description "Shouted",\n arguments (value ANY TEXT),\n' + " returns TYPE OF value,\n);\n\nUPPER(@value)\n" + ), + "macros/sql/month_of.sql": ( + 'MACRO (\n description "Month start",\n arguments (value ANY TEMPORAL),\n' + " returns TYPE OF value,\n);\n\nDATE_TRUNC('month', @value)\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_call_of_each_family_in_several_positions", + files={ + **_ROUNDED, + **contract_model( + "orders", + "amount (type BIGINT), loud (type VARCHAR), month (type TIMESTAMP), " + "in_cte (type DOUBLE)", + "WITH c AS (\n" + " SELECT o.order_id, @sql.rounded(CAST(o.amount_cents AS DOUBLE)) AS in_cte\n" + ' FROM __ref("stg_orders") AS o\n)\n' + "SELECT @sql.rounded(o.amount_cents, 0) AS amount,\n" + " @sql.shouted(o.status) AS loud,\n" + " @sql.month_of(o.created_at) AS month,\n" + " c.in_cte\n" + 'FROM __ref("stg_orders") AS o JOIN c ON c.order_id = o.order_id\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="an_unknown_argument_type_is_accepted", + files={ + **_ROUNDED, + **orders_model( + "SELECT @sql.rounded(bar(o.amount_cents, 0, 10)) AS v " + 'FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=0, + expected_fragments=("0 errors",), + ), + SqlMacroCompileTestCase( + description="a_wrong_family_argument_is_reported_at_the_call", + files={ + **_ROUNDED, + **orders_model( + 'SELECT @sql.rounded(o.status) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "argument value of macro rounded() expects a number; o.status is VARCHAR", + "models/marts/orders.sql:3:", + ), + ), + SqlMacroCompileTestCase( + description="a_wrong_family_literal_is_reported_at_the_call", + files={ + **_ROUNDED, + **orders_model("SELECT @sql.shouted(12) AS v\n"), + }, + expected_returncode=1, + expected_fragments=( + "error[P017]: argument value of macro shouted() expects text; 12 is an integer", + ), + ), + SqlMacroCompileTestCase( + description="a_body_wrong_for_its_family_is_reported_at_the_macro_file", + files={ + "macros/sql/m.sql": ( + 'MACRO (\n description "Shouted number",\n arguments (value ANY NUMERIC),\n' + ");\n\nUPPER(@value)\n" + ), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P025]: body of macro m() does not accept every value of value ANY NUMERIC", + "--> macros/sql/m.sql:6:1", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_any_type_families_when_compiling_then_calls_and_bodies_are_checked( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +def test_given_a_changed_returns_form_when_compiling_warm_then_the_cache_is_not_reused( + tmp_path: Path, +) -> None: + project_dir, cold = run_project( + tmp_path=tmp_path, + files={ + **_FIRST_OF, + **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), + }, + command=("compile",), + ) + macro: Path = project_dir / "macros/sql/first_of.sql" + macro.write_text(_FIRST_OF_UNTYPED["macros/sql/first_of.sql"], encoding="utf-8") + warm: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + assert "0 errors, 0 warnings" in output(cold), output(cold) + assert "warning[K003]: column 'day_value'" in output(warm), output(warm) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index 2e0e0b5f98..a10d18b03e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -788,6 +788,42 @@ def test_given_unwritable_cache_when_testing_since_then_warns_and_succeeds( unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), git_after_edits=("commit", "--quiet", "--all", "-m", "edit"), ), + SinceSelectionE2ETestCase( + description="changing a sql macro's returns form selects the test of its caller", + edits={ + _OUTER_SQL_MACRO: _SQL_MACROS[_OUTER_SQL_MACRO].replace( + "returns DOUBLE", "returns TYPE OF cents" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed", + "test_order_net\n executes macro net_dollars (changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="naming the CTEs of a CTE-list sql macro call selects the caller's test", + edits={ + "models/_macros/sql/live_orders.sql": ( + "MACRO (\n description 'Orders still open.',\n" + " arguments (src RELATION (order_id INTEGER, status VARCHAR)),\n);\n\n" + "WITH live AS (SELECT * FROM @src WHERE status = 'open')\n" + ), + "models/order_live.sql": ( + "MODEL (description 'Live orders.');\n\n" + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.live_orders(raw, __cte := open_orders)\n" + "SELECT order_id FROM open_orders\n" + ), + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=("Changed since main: 1 model, 1 macro added",), + unexpected_fragments=("FAIL=1",), + ), SinceSelectionE2ETestCase( description="reformatting, commenting and redescribing a sql macro changes nothing", edits={ From ea4a71691eff46435c1f343db323e51946ff44f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:36:39 +0100 Subject: [PATCH 08/28] fix: re-type models with typed markers once their inputs are known A marker asserted its type while an upstream star model's columns were still untyped, so a known column inside a call took the declared type. Such models are now enriched once their inputs are known, and the enrichment's types replace the earlier assertions. --- .../analysis_session/_helpers/session.rs | 34 ++++++++++++++++--- .../tests/test_typed_markers.rs | 7 +++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 69a0f8c235..a0534f82e3 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -410,14 +410,27 @@ impl AnalysisSession { .iter() .flatten() .any(|column| column.data_type.is_none()); - if !required.is_empty() && (untyped || star_pending) && !has_set_operation { + // A typed marker asserts its type before upstream star columns are typed, so its + // model is re-typed once every input is known. + let marked: bool = inputs_known + && self.request.models[model] + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); + let analysis: &ModelAnalysis = &self.outcome_mut(model)?.analysis; + if !required.is_empty() + && (untyped || star_pending || marked && analysis.analysis_succeeded) + && !has_set_operation + { candidates.push((model, input_schemas, star_pending)); } } let enrichments: Vec = self.native_enrichments(&candidates)?; for ((model, _, star_pending), native) in candidates.into_iter().zip(enrichments) { + let marked: bool = self.request.models[model] + .query_sql + .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); let outcome: &mut ModelOutcome = self.outcome_mut(model)?; - outcome.analysis = enriched_analysis(&outcome.analysis, native, star_pending); + outcome.analysis = enriched_analysis(&outcome.analysis, native, (star_pending, marked)); } Ok(()) } @@ -813,7 +826,7 @@ fn legacy_outcome( fn enriched_analysis( analysis: &ModelAnalysis, enriched: Enrichment, - star_pending: bool, + (star_pending, marked): (bool, bool), ) -> ModelAnalysis { let mut recovered: HashMap = HashMap::new(); for column in enriched @@ -826,6 +839,17 @@ fn enriched_analysis( recovered.insert(column.name.clone(), data_type.clone()); } } + // A marked model's enrichment knows every input, so its types replace earlier assertions. + let retyped: HashMap = if marked && enriched.analysis_succeeded { + enriched + .columns + .iter() + .flatten() + .filter_map(|column| Some((column.name.clone(), column.data_type.clone()?))) + .collect() + } else { + HashMap::new() + }; let star_expanded: bool = star_pending && enriched.analysis_succeeded; let replaced: bool = (!has_columns(analysis.columns.as_ref()) || star_expanded) && enriched.analysis_succeeded; @@ -844,7 +868,9 @@ fn enriched_analysis( }; let mut columns: Vec = merged.columns.take().unwrap_or_default(); for column in &mut columns { - if column.data_type.as_deref().is_none_or(str::is_empty) { + if let Some(data_type) = retyped.get(&column.name) { + column.data_type = Some(data_type.clone()); + } else if column.data_type.as_deref().is_none_or(str::is_empty) { column.data_type = recovered.get(&column.name).cloned(); } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs index 81053e52bb..6cabc808e0 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs @@ -36,7 +36,7 @@ fn column_v(lines: &[Vec]) -> String { .unwrap_or_else(|| format!("no v in {:?}", lines[0])) } -const CASES: [TypedMarkerTestCase; 6] = [ +const CASES: [TypedMarkerTestCase; 7] = [ TypedMarkerTestCase { description: "an unknown body takes the asserted type", sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", @@ -47,6 +47,11 @@ const CASES: [TypedMarkerTestCase; 6] = [ sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", expected_column: "v INT ", }, + TypedMarkerTestCase { + description: "a known aliased column keeps its inferred type", + sql: "SELECT __polyglot_typed(o.order_id, 'VARCHAR') AS v FROM __ref(\"staged\") AS o", + expected_column: "v INT ", + }, TypedMarkerTestCase { description: "a later set-operation branch types the union", sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ From 65e16f051815f7907dce0efe89b456c3a9b37610 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:36:39 +0100 Subject: [PATCH 09/28] test: cover SQL macro call typing in every query position --- .../test_sql_macro_typing_matrix.py | 392 ++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py new file mode 100644 index 0000000000..da75e0cdcd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -0,0 +1,392 @@ +"""E2E matrix of SQL macro calls typed by their returns in every position a call can stand.""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + BRACKET, + contract_model, + orders_model, + output, + run_project, + text_macro, + with_rules, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@dataclass(frozen=True) +class TypingPositionTestCase: + """One position of a call in a contracted model, and whether a known body type disagrees.""" + + description: str + sql: str + declared: str = "VARCHAR" + known_disagrees: bool = True + + +_FROM: str = 'FROM __ref("stg_orders") AS o' +_MACROS: dict[str, str] = { + **text_macro("status_bar", "VARCHAR", "bar(LENGTH(@value), 0, 10)"), + **text_macro("same", "VARCHAR", "@value", argument="ANY"), + **text_macro("shouted", "VARCHAR", "UPPER(@value)"), +} +_POSITIONS: tuple[TypingPositionTestCase, ...] = ( + TypingPositionTestCase("main_whole_item", f"SELECT {{m}} AS x {_FROM}\n"), + TypingPositionTestCase( + "cte_item_read_by_name", f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT x FROM c\n" + ), + TypingPositionTestCase( + "star_from_a_cte", f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT * FROM c\n" + ), + TypingPositionTestCase("derived_table", f"SELECT s.x FROM (SELECT {{m}} AS x {_FROM}) AS s\n"), + TypingPositionTestCase( + "scalar_subquery", f"SELECT (SELECT {{m}} {_FROM} LIMIT 1) AS x\n", known_disagrees=False + ), + TypingPositionTestCase( + "nested_with", + f"WITH c AS (WITH d AS (SELECT {{m}} AS x {_FROM}) SELECT x FROM d)\nSELECT x FROM c\n", + ), + TypingPositionTestCase( + "union_right_branch", + f"SELECT o.status AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + known_disagrees=False, + ), + TypingPositionTestCase( + "union_left_branch", + f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT o.status AS x {_FROM}\n", + known_disagrees=False, + ), + TypingPositionTestCase( + "union_both_branches", + f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + ), + TypingPositionTestCase( + "parenthesised_branch", + f"SELECT CAST(NULL AS VARCHAR) AS x {_FROM}\nUNION ALL\n(SELECT {{m}} AS x {_FROM})\n", + known_disagrees=False, + ), + TypingPositionTestCase( + "after_a_star_modifier", + "WITH c AS (SELECT o.order_id, o.status " + _FROM + ")\n" + "SELECT * EXCLUDE (order_id), {m} AS x FROM c AS o\n", + ), + TypingPositionTestCase( + "arithmetic", + f"SELECT LENGTH({{m}}) + 1 AS x {_FROM}\n", + declared="BIGINT", + known_disagrees=False, + ), + TypingPositionTestCase( + "coalesce", f"SELECT COALESCE({{m}}, {{m}}) AS x {_FROM}\n", known_disagrees=False + ), + TypingPositionTestCase( + "case", + f"SELECT CASE WHEN o.order_id > 1 THEN {{m}} ELSE NULL END AS x {_FROM}\n", + known_disagrees=False, + ), + TypingPositionTestCase( + "function_argument", + f"SELECT LENGTH({{m}}) AS x {_FROM}\n", + declared="BIGINT", + known_disagrees=False, + ), + TypingPositionTestCase( + "aggregate_over_the_call", f"SELECT MAX({{m}}) AS x {_FROM}\n", known_disagrees=False + ), + TypingPositionTestCase( + "window", + f"SELECT MAX({{m}}) OVER (ORDER BY o.order_id) AS x {_FROM}\n", + known_disagrees=False, + ), + TypingPositionTestCase("unaliased_call", f"SELECT {{m}} {_FROM}\n", known_disagrees=False), +) + + +def _model(position: TypingPositionTestCase, call: str) -> dict[str, str]: + sql: str = position.sql.replace("{m}", call) + if position.description == "unaliased_call": + return orders_model(sql) + if position.description == "after_a_star_modifier": + return { + **orders_model(sql), + **contract_model( + "order_x", f"x (type {position.declared})", 'SELECT b.x FROM __ref("orders") AS b\n' + ), + } + return contract_model("orders", f"x (type {position.declared})", sql) + + +@pytest.mark.parametrize("position", _POSITIONS, ids=lambda case: case.description) +def test_given_an_unknown_body_when_compiling_then_the_call_takes_its_returns_type( + position: TypingPositionTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, + files={**_MACROS, **_model(position, "@sql.status_bar(o.status)")}, + command=("compile", "--no-cache"), + ) + + assert result.returncode == 0, output(result) + assert "0 errors, 0 warnings" in output(result), output(result) + + +@pytest.mark.parametrize( + "position", + [position for position in _POSITIONS if position.known_disagrees], + ids=lambda case: case.description, +) +def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stands( + position: TypingPositionTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, + files={**_MACROS, **_model(position, "@sql.same(o.order_id)")}, + command=("compile", "--no-cache"), + ) + + assert "error[K002]: column 'x' inferred as" in output(result), output(result) + + +@dataclass(frozen=True) +class TypingCaseTestCase: + """One project and what compiling it prints.""" + + description: str + files: dict[str, str] + expected_returncode: int + expected_fragments: tuple[str, ...] + + +@pytest.mark.parametrize( + "test_case", + [ + TypingCaseTestCase( + description="aggregate_inside_the_body", + files={ + **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), + **contract_model( + "orders", + "status (type VARCHAR), x (type BIGINT)", + f"SELECT o.status, @sql.summed(o.status) AS x {_FROM}\nGROUP BY o.status\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="aggregate_inside_the_body_still_needs_group_by", + files={ + **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), + **orders_model(f"SELECT o.status, @sql.summed(o.status) AS x {_FROM}\n"), + }, + expected_returncode=1, + expected_fragments=("error[B230]",), + ), + TypingCaseTestCase( + description="a_nested_call_types_the_macro_calling_it", + files={ + **_MACROS, + **text_macro("outer_bar", None, "@sql.status_bar(@value)"), + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.outer_bar(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="an_any_argument_of_unknown_type_takes_returns", + files={ + **_MACROS, + **contract_model( + "orders", + "x (type VARCHAR)", + f"SELECT @sql.same(bar(o.order_id, 0, 10)) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="a_known_identical_body", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.shouted(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="a_call_inside_a_python_macro_argument_is_inferred_only", + files={ + **_MACROS, + **BRACKET, + **contract_model( + "orders", + "x (type VARCHAR)", + f"SELECT @py.bracket(@sql.status_bar(o.status)) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="an_outer_cast_wins_and_satisfies_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **_MACROS, + **contract_model( + "orders", + "x (type BIGINT)", + f"SELECT CAST(@sql.status_bar(o.status) AS BIGINT) AS x {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + TypingCaseTestCase( + description="a_call_inside_arithmetic_needs_a_cast_for_the_cast_rule", + files={ + **with_rules("SQBRCONTRACT105"), + **_MACROS, + **contract_model( + "orders", + "x (type BIGINT)", + f"SELECT LENGTH(@sql.status_bar(o.status)) + 1 AS x {_FROM}\n", + ), + }, + expected_returncode=1, + expected_fragments=( + 'error[SQBRCONTRACT105]: output "x" is calculated without an outer explicit cast', + "Wrap the complete expression in CAST(... AS BIGINT)", + ), + ), + TypingCaseTestCase( + description="a_binding_error_after_a_call_is_located_where_it_was_written", + files={ + **_MACROS, + **orders_model(f"SELECT @sql.status_bar(o.status) AS x, o.missing {_FROM}\n"), + }, + expected_returncode=1, + expected_fragments=("--> models/marts/orders.sql:3:42",), + ), + TypingCaseTestCase( + description="the_reserved_analysis_function_is_rejected_in_a_model", + files={**orders_model(f"SELECT __polyglot_typed(o.status, 'VARCHAR') AS x {_FROM}\n")}, + expected_returncode=1, + expected_fragments=("uses __polyglot_typed, a name reserved for SQL analysis",), + ), + TypingCaseTestCase( + description="a_downstream_model_sees_the_returns_type", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" + ), + **contract_model( + "order_labels", "x (type BIGINT)", 'SELECT b.x FROM __ref("orders") AS b\n' + ), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'x' inferred as",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_typed_call_case_when_compiling_then_it_reports_as_expected( + test_case: TypingCaseTestCase, tmp_path: Path +) -> None: + _, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") + ) + + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) + + +@pytest.mark.xfail( + reason="a * REPLACE item takes its source column's type; a known polyglot gap", strict=True +) +def test_given_a_call_in_a_star_replace_when_compiling_then_it_takes_its_returns_type( + tmp_path: Path, +) -> None: + _, result = run_project( + tmp_path=tmp_path, + files={ + **_MACROS, + **contract_model( + "orders", + "order_id (type VARCHAR)", + "WITH c AS (SELECT o.order_id " + _FROM + ")\n" + "SELECT * REPLACE (@sql.status_bar(CAST(order_id AS VARCHAR)) AS order_id) " + "FROM c\n", + ), + }, + command=("compile", "--no-cache"), + ) + + assert "0 errors, 0 warnings" in output(result), output(result) + + +def test_given_typed_calls_when_compiling_then_the_compiled_sql_has_no_marker( + tmp_path: Path, +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" + ), + }, + command=("compile", "--no-cache"), + ) + compiled: list[Path] = [ + path for path in (project_dir / "target").rglob("orders.sql") if "compiled" in str(path) + ] + + assert result.returncode == 0, output(result) + assert compiled, output(result) + text: str = compiled[0].read_text(encoding="utf-8") + assert "bar(LENGTH(o.status), 0, 10) AS x" in text, text + assert "__polyglot" not in text, text + + +def test_given_a_call_when_reading_lineage_then_it_matches_the_inline_sql(tmp_path: Path) -> None: + macro_dir, _ = run_project( + tmp_path=tmp_path / "macro", + files={ + **text_macro("ident", "BIGINT", "@value", argument="BIGINT"), + **orders_model(f"SELECT @sql.ident(o.order_id) AS x {_FROM}\n"), + }, + command=("compile", "--no-cache"), + ) + inline_dir, _ = run_project( + tmp_path=tmp_path / "inline", + files=orders_model(f"SELECT o.order_id AS x {_FROM}\n"), + command=("compile", "--no-cache"), + ) + lineage = ("--no-color", "lineage", "orders.x", "--format", "list") + through_macro: subprocess.CompletedProcess[str] = run_sqb( + command=lineage, project_dir=macro_dir + ) + inline: subprocess.CompletedProcess[str] = run_sqb(command=lineage, project_dir=inline_dir) + + assert through_macro.returncode == 0, output(through_macro) + assert "stg_orders.order_id -> orders.x" in through_macro.stdout, output(through_macro) + assert _edges(through_macro.stdout) == _edges(inline.stdout) + + +def _edges(printed: str) -> list[str]: + """The lineage edges a list-format lineage command printed.""" + + return [line.strip() for line in printed.splitlines() if "->" in line] From 1c5fb1e1138a706180eef14b5f545a37c528a941 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:37:34 +0100 Subject: [PATCH 10/28] docs: describe SQL macro result types, TYPE OF, ANY families and __cte --- .../docs/concepts/macros/sql-macros.md | 101 ++++++++++++++---- .../references/docs/concepts/testing.md | 4 +- .../references/macros-and-declarations.md | 8 +- .../docs/docs/concepts/macros/sql-macros.mdx | 97 +++++++++++++---- .../content/docs/docs/concepts/testing.mdx | 4 +- 5 files changed, 174 insertions(+), 40 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index 82417bf04e..b0b2801910 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -13,7 +13,9 @@ Online: https://sqlbuild.com/docs/concepts/macros/sql-macros/ - Parameter types - Parameters in the body - Compose macros -- Checks and tools +- Checks +- Result types +- Tools A SQL macro is a reusable piece of SQL with typed parameters. Unlike a [Python macro](python-macros.md), it holds no logic: its body is SQL, so the compiler checks @@ -64,10 +66,11 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` -`returns` is required for expression macros and not allowed for the other forms. +`returns` is optional for expression macros and not allowed for the other forms. See +[Result types](#result-types) for what it does. The description follows the same rules as every header string: write text that contains double quotes as `"""..."""`, and a wrapped description is stored folded into one flowing text (see @@ -93,7 +96,7 @@ compiles to: ```sql SELECT o.order_id, - ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees + CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM analytics.stg_orders AS o ``` @@ -131,6 +134,27 @@ The compiled model holds `orders_raw`, `live`, `ranked`, `latest` and `final`, f order, and later CTEs may read any of the macro's CTEs. Commas around the call are optional; `sqb format` normalises them. +The last CTE of a CTE-list macro is its result. To call one macro more than once in a model, name +each call's CTEs with the reserved `__cte` argument: + +```sql +WITH eu_raw AS (SELECT * FROM __ref("stg_orders_eu")), +us_raw AS (SELECT * FROM __ref("stg_orders_us")), +@sql.clean_orders(eu_raw, __cte := eu_orders) +@sql.clean_orders(us_raw, __cte := us_orders) +final AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders) +SELECT * FROM final +``` + +The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so +this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, +`us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are +renamed with them; arguments are never renamed, even one named like a macro CTE. Later CTEs may read +the prefixed ones too. + +`__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in +`arguments`. A call without it keeps the macro's own names. + ### Column lists A column-list macro inserts its columns into the caller's select list. Every column needs a name: @@ -152,7 +176,8 @@ Use a column list when the same ordered set of columns repeats, for example in e `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. A CTE or column name a macro inserts that the caller already defines fails with `P018`, which names -both. Nothing is renamed, so a model cannot call the same CTE-list macro twice. +both. Calling the same CTE-list macro twice without `__cte` fails the same way, and the error +suggests `__cte`. ## Parameter types @@ -162,6 +187,7 @@ Every parameter has a type, and may have a `DEFAULT`: |---|---| | `BIGINT`, `VARCHAR`, `TIMESTAMP`, ... | Any SQL expression of a compatible type, evaluated per row: a column, arithmetic, a function call, a literal or a constant. | | `ANY` | Any scalar SQL expression, whatever its type: a column, a literal, an expression or a scalar constant. Its type is not checked. Lists and relations fail with `P017`. | +| `ANY NUMERIC`, `ANY TEXT`, `ANY TEMPORAL` | Like `ANY`, but only values of one family: integer, decimal and float types; string types; or date, time and timestamp types. | | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | @@ -179,6 +205,19 @@ still checked, and the argument's type counts as unknown there. `ANY` is not `VA value of any type. `ANY` takes a `DEFAULT` but not `CONST`, which needs a type to check values against, and it cannot be a `LIST` or `SET` element type. +### Choosing an argument type + +- A fixed type, such as `BIGINT`, when the body needs that type. Callers may pass any compatible + value, and the body is checked with that type. +- A family, such as `ANY NUMERIC`, when the body works for every type of the family, such as + `ROUND(@value, @places)`. A call whose argument has a known type outside the family fails with + `P017` at the argument. The body is checked as if the argument had the family's widest type + (`DOUBLE`, `VARCHAR` or `TIMESTAMP`), so `UPPER(@value)` on `ANY NUMERIC` fails with `P025` at the + macro file. Pair a family with `returns TYPE OF value` to return the caller's own type. +- `ANY` when the body works for values of any type, typically because it casts the argument itself. + +Families take a `DEFAULT` but not `CONST`, and are not `LIST` or `SET` element types. + Leave `CONST` off unless the body needs a literal. `CONST` is part of the macro's contract: callers can no longer pass a column. @@ -207,7 +246,7 @@ These fail with `P022`: - a relation parameter written bare where it would read a table, such as `FROM src` or `src.created_at`; the error says `src is a parameter of macro clean_orders(); write @src` -Parameters may not be named `sql`, `py`, `const`, `enum` or `param`. +Parameters may not be named `sql`, `py`, `const`, `enum`, `param` or `__cte`. ### Positions that need a literal @@ -256,7 +295,7 @@ isn't visible from there fails with `S026`; to use a caller's value, make it an `@const(...)` at the call site. SQL macros a body calls resolve the same way, so a macro can call a folder-local macro beside it that its callers cannot see. -## Checks and tools +## Checks Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: @@ -282,18 +321,42 @@ parameters of a calling macro, and calls in models with runtime placeholders; th column checks still run on the expanded SQL. All other errors also point at the call and at the macro file. -A select-list item that is one expression macro call under an alias, as in -`@sql.cents_to_dollars(o.amount_cents) AS amount`, has the macro's `returns` type wherever the -compiler infers no type for the expanded SQL, or exactly that type. This holds in the main query -and in any top-level CTE, including each branch of their `UNION`, `INTERSECT` or `EXCEPT`, and the -type flows on as an inferred type would: through `SELECT * FROM final`, pass-through columns and -joins, to contract checks (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A -set operation's column combines its branches as usual, so a sibling branch of another type still -leaves it unproven, and a CTE holding one is read by column name, since `SELECT *` over it is not -typed. Such a call needs no `CAST` to satisfy a matching contract column. Any other -inferred type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and `K002` -reports the mismatch. A call inside a larger expression, a subquery, a nested `WITH` or a set -operation with a parenthesised branch is typed from the expanded SQL as before. +## Result types + +An expression macro call is typed wherever it stands, from its expanded SQL, as if the SQL were +written there. `returns` declares the type for when that SQL's type is unknown: + +```sql +MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); + +bar(LENGTH(@value), 0, 10) +``` + +- The declared type applies only where the expanded SQL's type is unknown, or partly unknown, such + as an array of unknown elements. A known type stands, even a close one such as `VARCHAR(100)` for + `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. +- The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each + branch of a set operation, through operators and functions around the call, to contract checks + (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A select-list item that is + one call to a macro with `returns` needs no `CAST` for `SQBRCONTRACT105`; a call inside a larger + expression still does. +- Without `returns`, the call is typed by inference alone. When an output is one call whose type is + unknown, `K003` and `SQBRCONTRACT105` suggest declaring `returns` on the macro. +- `returns TYPE OF ` declares the type of that argument at each call, under the same + rule: `arguments (value ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF value`. If the + argument's own type is unknown, so is the call's. The argument must be a scalar argument of the + macro (`P015`). +- A fixed `returns` is checked against the body at the macro file. The body is typed from the + declared argument types; a known type of another family, such as `UPPER(@value)` with + `returns BIGINT`, fails with `P024`. A body whose type cannot be inferred, for example because it + reads `ANY` arguments or calls a function of unknown type, trusts `returns`. +- A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed + by inference alone. +- `returns` never changes the compiled SQL. + +A `* REPLACE` item takes its source column's type rather than the call's. + +## Tools - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 7e5146c01f..6ef7862a67 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -615,7 +615,9 @@ __macro_actual__ AS (SELECT order_id, updated_at FROM latest), __macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at) ``` -Other test modes may not call a CTE-list macro in their `WITH` list. +These calls may pass `__cte := name`, as in a model, to call the macro more than once or to give +its CTEs names of the test's own. Other test modes may not call a CTE-list macro in their `WITH` +list. ### UDF tests diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md index dc17d12f6f..52bb4c35de 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md @@ -28,7 +28,7 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` ```sql @@ -36,6 +36,12 @@ SELECT order_id, @sql.cents_to_dollars(amount_cents) AS amount_dollars FROM __ref("stg_orders") ``` +`returns` is optional: a call is typed from its expanded SQL, and `returns` (or +`returns TYPE OF `) applies only where that type is unknown. A `returns` whose body has a +known type of another family fails with `P024`. Narrow `ANY` arguments with `ANY NUMERIC`, +`ANY TEXT` or `ANY TEMPORAL`, and call a CTE-list macro more than once in a model with +`__cte := name`. + Full reference: [docs/concepts/macros/sql-macros.md](docs/concepts/macros/sql-macros.md). **Python macros** live in `macros/python/` and are plain Python functions that return final SQL diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 58da2a1492..f7d0f0b30c 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -52,10 +52,11 @@ MACRO ( returns DECIMAL(18, 2), ); -ROUND(CAST(@amount_cents AS DECIMAL(18, 4)) / 100, @places) +CAST(ROUND(@amount_cents / 100, @places) AS DECIMAL(18, 2)) ``` -`returns` is required for expression macros and not allowed for the other forms. +`returns` is optional for expression macros and not allowed for the other forms. See +[Result types](#result-types) for what it does. The description follows the same rules as every header string: write text that contains double quotes as `"""..."""`, and a wrapped description is stored folded into one flowing text (see @@ -81,7 +82,7 @@ compiles to: ```sql SELECT o.order_id, - ROUND(CAST((o.fee_cents + o.tax_cents) AS DECIMAL(18, 4)) / 100, 0) AS fees + CAST(ROUND((o.fee_cents + o.tax_cents) / 100, 0) AS DECIMAL(18, 2)) AS fees FROM analytics.stg_orders AS o ``` @@ -119,6 +120,27 @@ The compiled model holds `orders_raw`, `live`, `ranked`, `latest` and `final`, f order, and later CTEs may read any of the macro's CTEs. Commas around the call are optional; `sqb format` normalises them. +The last CTE of a CTE-list macro is its result. To call one macro more than once in a model, name +each call's CTEs with the reserved `__cte` argument: + +```sql +WITH eu_raw AS (SELECT * FROM __ref("stg_orders_eu")), +us_raw AS (SELECT * FROM __ref("stg_orders_us")), +@sql.clean_orders(eu_raw, __cte := eu_orders) +@sql.clean_orders(us_raw, __cte := us_orders) +final AS (SELECT * FROM eu_orders UNION ALL SELECT * FROM us_orders) +SELECT * FROM final +``` + +The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so +this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, +`us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are +renamed with them; arguments are never renamed, even one named like a macro CTE. Later CTEs may read +the prefixed ones too. + +`__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in +`arguments`. A call without it keeps the macro's own names. + ### Column lists A column-list macro inserts its columns into the caller's select list. Every column needs a name: @@ -140,7 +162,8 @@ Use a column list when the same ordered set of columns repeats, for example in e `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. A CTE or column name a macro inserts that the caller already defines fails with `P018`, which names -both. Nothing is renamed, so a model cannot call the same CTE-list macro twice. +both. Calling the same CTE-list macro twice without `__cte` fails the same way, and the error +suggests `__cte`. ## Parameter types @@ -150,6 +173,7 @@ Every parameter has a type, and may have a `DEFAULT`: |---|---| | `BIGINT`, `VARCHAR`, `TIMESTAMP`, ... | Any SQL expression of a compatible type, evaluated per row: a column, arithmetic, a function call, a literal or a constant. | | `ANY` | Any scalar SQL expression, whatever its type: a column, a literal, an expression or a scalar constant. Its type is not checked. Lists and relations fail with `P017`. | +| `ANY NUMERIC`, `ANY TEXT`, `ANY TEMPORAL` | Like `ANY`, but only values of one family: integer, decimal and float types; string types; or date, time and timestamp types. | | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | @@ -167,6 +191,19 @@ still checked, and the argument's type counts as unknown there. `ANY` is not `VA value of any type. `ANY` takes a `DEFAULT` but not `CONST`, which needs a type to check values against, and it cannot be a `LIST` or `SET` element type. +### Choosing an argument type + +- A fixed type, such as `BIGINT`, when the body needs that type. Callers may pass any compatible + value, and the body is checked with that type. +- A family, such as `ANY NUMERIC`, when the body works for every type of the family, such as + `ROUND(@value, @places)`. A call whose argument has a known type outside the family fails with + `P017` at the argument. The body is checked as if the argument had the family's widest type + (`DOUBLE`, `VARCHAR` or `TIMESTAMP`), so `UPPER(@value)` on `ANY NUMERIC` fails with `P025` at the + macro file. Pair a family with `returns TYPE OF value` to return the caller's own type. +- `ANY` when the body works for values of any type, typically because it casts the argument itself. + +Families take a `DEFAULT` but not `CONST`, and are not `LIST` or `SET` element types. + Leave `CONST` off unless the body needs a literal. `CONST` is part of the macro's contract: callers can no longer pass a column. @@ -195,7 +232,7 @@ These fail with `P022`: - a relation parameter written bare where it would read a table, such as `FROM src` or `src.created_at`; the error says `src is a parameter of macro clean_orders(); write @src` -Parameters may not be named `sql`, `py`, `const`, `enum` or `param`. +Parameters may not be named `sql`, `py`, `const`, `enum`, `param` or `__cte`. ### Positions that need a literal @@ -244,7 +281,7 @@ isn't visible from there fails with `S026`; to use a caller's value, make it an `@const(...)` at the call site. SQL macros a body calls resolve the same way, so a macro can call a folder-local macro beside it that its callers cannot see. -## Checks and tools +## Checks Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: @@ -270,18 +307,42 @@ parameters of a calling macro, and calls in models with runtime placeholders; th column checks still run on the expanded SQL. All other errors also point at the call and at the macro file. -A select-list item that is one expression macro call under an alias, as in -`@sql.cents_to_dollars(o.amount_cents) AS amount`, has the macro's `returns` type wherever the -compiler infers no type for the expanded SQL, or exactly that type. This holds in the main query -and in any top-level CTE, including each branch of their `UNION`, `INTERSECT` or `EXCEPT`, and the -type flows on as an inferred type would: through `SELECT * FROM final`, pass-through columns and -joins, to contract checks (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A -set operation's column combines its branches as usual, so a sibling branch of another type still -leaves it unproven, and a CTE holding one is read by column name, since `SELECT *` over it is not -typed. Such a call needs no `CAST` to satisfy a matching contract column. Any other -inferred type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and `K002` -reports the mismatch. A call inside a larger expression, a subquery, a nested `WITH` or a set -operation with a parenthesised branch is typed from the expanded SQL as before. +## Result types + +An expression macro call is typed wherever it stands, from its expanded SQL, as if the SQL were +written there. `returns` declares the type for when that SQL's type is unknown: + +```sql +MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); + +bar(LENGTH(@value), 0, 10) +``` + +- The declared type applies only where the expanded SQL's type is unknown, or partly unknown, such + as an array of unknown elements. A known type stands, even a close one such as `VARCHAR(100)` for + `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. +- The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each + branch of a set operation, through operators and functions around the call, to contract checks + (`K002`, `K003`), `SQBRCONTRACT105` and models that read the column. A select-list item that is + one call to a macro with `returns` needs no `CAST` for `SQBRCONTRACT105`; a call inside a larger + expression still does. +- Without `returns`, the call is typed by inference alone. When an output is one call whose type is + unknown, `K003` and `SQBRCONTRACT105` suggest declaring `returns` on the macro. +- `returns TYPE OF ` declares the type of that argument at each call, under the same + rule: `arguments (value ANY NUMERIC, places INT CONST DEFAULT 2), returns TYPE OF value`. If the + argument's own type is unknown, so is the call's. The argument must be a scalar argument of the + macro (`P015`). +- A fixed `returns` is checked against the body at the macro file. The body is typed from the + declared argument types; a known type of another family, such as `UPPER(@value)` with + `returns BIGINT`, fails with `P024`. A body whose type cannot be inferred, for example because it + reads `ANY` arguments or calls a function of unknown type, trusts `returns`. +- A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed + by inference alone. +- `returns` never changes the compiled SQL. + +A `* REPLACE` item takes its source column's type rather than the call's. + +## Tools - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 604fe7f6d1..b51229cdf8 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -590,7 +590,9 @@ __macro_actual__ AS (SELECT order_id, updated_at FROM latest), __macro_expected__ AS (SELECT 1 AS order_id, TIMESTAMP '2026-01-02' AS updated_at) ``` -Other test modes may not call a CTE-list macro in their `WITH` list. +These calls may pass `__cte := name`, as in a model, to call the macro more than once or to give +its CTEs names of the test's own. Other test modes may not call a CTE-list macro in their `WITH` +list. ### UDF tests From f87cb955660e0b0e9d964f0dc22330c880c41886 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 07:08:32 +0100 Subject: [PATCH 11/28] build: point polyglot at the typed markers rebased onto upstream 0.13.4 --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- .../src/sql_quality/_helpers/removal_safety.rs | 11 +++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd814d663c..6263de95b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" +source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" +source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=bf017ad89c558f9876d6474e10251b84b539065a#bf017ad89c558f9876d6474e10251b84b539065a" +source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 81d7282542..c68a46aea3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "bf017ad89c558f9876d6474e10251b84b539065a", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "ee40a7246867f6da80fcf38b932c1eb5886a415e", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "bf017ad89c558f9876d6474e10251b84b539065a", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "ee40a7246867f6da80fcf38b932c1eb5886a415e", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs index 0de0113fd9..eb5d8398a2 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/removal_safety.rs @@ -76,11 +76,14 @@ fn orders_by_all(select: &Select) -> bool { .order_by .iter() .flat_map(|order| order.expressions.iter()) - .any(|ordered| { - matches!(&ordered.this, Expression::Column(column) - if column.table.is_none() + .any(|ordered| match &ordered.this { + Expression::Column(column) => { + column.table.is_none() && !column.name.quoted - && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL)) + && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL) + } + Expression::Var(var) => var.this.eq_ignore_ascii_case(ORDER_BY_ALL), + _ => false, }) } From 69e7102dfcd829ba1b4fcf46862de20ab485800c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:13:18 +0100 Subject: [PATCH 12/28] refactor: fit SQL macro typing to the architecture and test-shape checks --- .../analysis_session/_helpers/session.rs | 8 +- .../analysis_session/tests/helpers.rs | 27 ++ .../tests/test_typed_markers.rs | 168 ++++---- .../analysis_session/tests/test_types.rs | 6 + .../_helpers/compatibility_types.rs | 4 +- .../main/annotated_output_types.rs | 40 ++ .../src/query_analysis/main/mod.rs | 1 + .../src/query_analysis/main/output_types.rs | 34 +- .../src/sql_macros/_helpers/mod.rs | 16 +- .../sql_macros/_helpers/{ => parsing}/body.rs | 8 +- .../_helpers/{ => parsing}/definition.rs | 18 +- .../_helpers/{ => parsing}/header.rs | 4 +- .../_helpers/{ => parsing}/lexing.rs | 0 .../{ => parsing}/literal_positions.rs | 4 +- .../src/sql_macros/_helpers/parsing/mod.rs | 6 + .../_helpers/{ => parsing}/parameters.rs | 2 +- .../_helpers/{ => rendering}/arguments.rs | 8 +- .../_helpers/{ => rendering}/cte_rename.rs | 21 +- .../_helpers/{ => rendering}/expansion.rs | 26 +- .../src/sql_macros/_helpers/rendering/mod.rs | 4 + .../_helpers/{ => rendering}/placement.rs | 8 +- .../src/sql_macros/_helpers/typing/mod.rs | 3 + .../_helpers/{ => typing}/probes.rs | 8 +- .../_helpers/{ => typing}/returns_check.rs | 116 +++--- .../_helpers/{ => typing}/typed_calls.rs | 7 +- .../src/sql_macros/constants.rs | 3 +- .../src/sql_macros/main/bare_macro_calls.rs | 2 +- .../src/sql_macros/main/byte_text_splices.rs | 2 +- .../main/check_sql_macro_arguments.rs | 4 +- .../sql_macros/main/expand_sql_macro_call.rs | 9 +- .../src/sql_macros/main/moved_typed_calls.rs | 2 +- .../src/sql_macros/main/parse_sql_macro.rs | 2 +- .../main/render_inline_collection.rs | 4 +- .../sql_macros/main/spliced_typed_calls.rs | 2 +- .../sql_macros/main/sql_macro_body_form.rs | 4 +- .../src/sql_macros/main/sql_macro_calls.rs | 2 +- .../main/sql_macro_parameter_spans.rs | 4 +- .../sql_macros/main/typed_analysis_pass.rs | 5 +- .../src/sql_macros/main/typed_analysis_sql.rs | 5 +- .../sql_macros/main/untyped_call_outputs.rs | 19 +- .../main/with_list_sql_macro_calls.rs | 4 +- .../src/sql_macros/tests/helpers.rs | 36 +- .../src/sql_macros/tests/test_typed_calls.rs | 130 ++++--- .../src/sql_macros/tests/test_types.rs | 23 ++ .../src/cursor_intrinsics/_helpers/mod.rs | 1 + .../cursor_intrinsics/_helpers/replacement.rs | 62 +++ .../src/cursor_intrinsics/main/mod.rs | 1 + .../main/replace_intrinsics.rs | 58 +-- .../main/replace_intrinsics_with_spans.rs | 13 + .../src/cursor_intrinsics/models.rs | 9 + .../_helpers/input_loops/model_loop.rs | 17 +- .../_helpers/render/sql_macro_host.rs | 15 +- .../main/compile/sql_macros/_test_types.py | 23 ++ .../main/compile/sql_macros/helpers.py | 40 +- .../sql_macros/test_sql_macro_returns.py | 171 +++++---- .../test_sql_macro_typing_matrix.py | 361 ++++++++++-------- 56 files changed, 907 insertions(+), 673 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/body.rs (98%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/definition.rs (96%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/header.rs (99%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/lexing.rs (100%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/literal_positions.rs (98%) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => parsing}/parameters.rs (97%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => rendering}/arguments.rs (98%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => rendering}/cte_rename.rs (95%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => rendering}/expansion.rs (96%) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => rendering}/placement.rs (98%) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => typing}/probes.rs (98%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => typing}/returns_check.rs (75%) rename crates/sqlbuild-analysis/src/sql_macros/_helpers/{ => typing}/typed_calls.rs (96%) create mode 100644 crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs create mode 100644 crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index a0534f82e3..19327a8c92 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -386,8 +386,7 @@ impl AnalysisSession { } let request: &ModelRequest = &self.request.models[model]; let required: Vec = request.required_names.clone(); - // A set operation is enriched only to type its SQL macro calls by their markers. - let has_set_operation: bool = request.has_set_operation + let unmarked_set_operation: bool = request.has_set_operation && !request .query_sql .contains(polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION); @@ -410,8 +409,6 @@ impl AnalysisSession { .iter() .flatten() .any(|column| column.data_type.is_none()); - // A typed marker asserts its type before upstream star columns are typed, so its - // model is re-typed once every input is known. let marked: bool = inputs_known && self.request.models[model] .query_sql @@ -419,7 +416,7 @@ impl AnalysisSession { let analysis: &ModelAnalysis = &self.outcome_mut(model)?.analysis; if !required.is_empty() && (untyped || star_pending || marked && analysis.analysis_succeeded) - && !has_set_operation + && !unmarked_set_operation { candidates.push((model, input_schemas, star_pending)); } @@ -839,7 +836,6 @@ fn enriched_analysis( recovered.insert(column.name.clone(), data_type.clone()); } } - // A marked model's enrichment knows every input, so its types replace earlier assertions. let retyped: HashMap = if marked && enriched.analysis_succeeded { enriched .columns diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 459254d0f7..8205ca8924 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -697,3 +697,30 @@ pub(crate) fn injected_relations( request.forced_order = Some((relations.dependency_ordered, relations.cyclic)); request } + +/// The request a fast-lineage contract compile sends for `typed` over `sql`, reading `staged`. +pub(crate) fn typed_marker_request(sql: &'static str) -> SessionRequest { + let staged: ModelSpec = ( + "staged", + "SELECT * FROM __source(\"raw_orders\")", + &["raw_orders"], + &[], + ); + let mut requests = model_requests(&[("typed", sql, &[], &["staged"]), staged]); + requests[0].recover_cte_facts = true; + let mut request: SessionRequest = orders_request(requests); + request.rich_type_inference = false; + let own = shapes(&[("typed", &[("v", "VARCHAR")])]); + request.column_types.extend(own.clone()); + request.complete_schemas.extend(own); + request +} + +/// The described line of the first model's column `v`. +pub(crate) fn described_column_v(lines: &[Vec]) -> String { + lines[0] + .iter() + .find(|line| line.starts_with("v ")) + .cloned() + .unwrap_or_else(|| format!("no v in {:?}", lines[0])) +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs index 6cabc808e0..8a8a270e5c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_typed_markers.rs @@ -1,108 +1,104 @@ use sqlbuild_cache::store::models::NativeStore; -use crate::assembly::analysis_session::models::SessionRequest; use crate::assembly::analysis_session::tests::helpers::{ - cached_run, completed, described, model_requests, orders_request, shapes, started, + cached_run, completed, described, described_column_v, started, typed_marker_request, }; use crate::assembly::analysis_session::tests::test_types::{ - CachedRun, ModelSpec, TypedMarkerTestCase, + CachedRun, TypedMarkerCacheTestCase, TypedMarkerTestCase, }; -const STAGED: ModelSpec = ( - "staged", - "SELECT * FROM __source(\"raw_orders\")", - &["raw_orders"], - &[], -); - -/// The orders request over `sql` reading `staged`, as a fast-lineage compile of a contract sends it. -fn request(sql: &'static str) -> SessionRequest { - let mut requests = model_requests(&[("typed", sql, &[], &["staged"]), STAGED]); - requests[0].recover_cte_facts = true; - let mut request: SessionRequest = orders_request(requests); - request.rich_type_inference = false; - let own = shapes(&[("typed", &[("v", "VARCHAR")])]); - request.column_types.extend(own.clone()); - request.complete_schemas.extend(own); - request -} - -/// The described line of the first model's column `v`. -fn column_v(lines: &[Vec]) -> String { - lines[0] - .iter() - .find(|line| line.starts_with("v ")) - .cloned() - .unwrap_or_else(|| format!("no v in {:?}", lines[0])) -} - -const CASES: [TypedMarkerTestCase; 7] = [ - TypedMarkerTestCase { - description: "an unknown body takes the asserted type", - sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", - expected_column: "v TEXT", - }, - TypedMarkerTestCase { - description: "a known body keeps its inferred type", - sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", - expected_column: "v INT ", - }, - TypedMarkerTestCase { - description: "a known aliased column keeps its inferred type", - sql: "SELECT __polyglot_typed(o.order_id, 'VARCHAR') AS v FROM __ref(\"staged\") AS o", - expected_column: "v INT ", - }, - TypedMarkerTestCase { - description: "a later set-operation branch types the union", - sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ - SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", - expected_column: "v TEXT", - }, - TypedMarkerTestCase { - description: "a NULL body takes the asserted type in a union", - sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"staged\") AS o UNION ALL \ - SELECT __polyglot_typed(NULL, 'BIGINT') AS v FROM __ref(\"staged\") AS o", - expected_column: "v BIGINT", - }, - TypedMarkerTestCase { - description: "a CTE read by name", - sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ - FROM __ref(\"staged\")) SELECT v FROM c", - expected_column: "v TEXT", - }, - TypedMarkerTestCase { - description: "an operand of a larger expression", - sql: "SELECT COALESCE(__polyglot_typed(bar(amount, 0, 10), 'VARCHAR'), 'x') AS v \ - FROM __ref(\"staged\")", - expected_column: "v TEXT", - }, -]; - #[test] fn given_typed_markers_when_analysing_then_each_call_has_its_asserted_or_inferred_type() { - for test_case in CASES { - let (_, outcomes, _) = completed(started(request(test_case.sql))); + let test_cases = [ + TypedMarkerTestCase { + description: "an unknown body takes the asserted type", + sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a known body keeps its inferred type", + sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v INT ", + }, + TypedMarkerTestCase { + description: "a known aliased column keeps its inferred type", + sql: "SELECT __polyglot_typed(o.order_id, 'VARCHAR') AS v FROM __ref(\"staged\") AS o", + expected_column: "v INT ", + }, + TypedMarkerTestCase { + description: "a later set-operation branch types the union", + sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ + SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "a NULL body takes the asserted type in a union", + sql: "SELECT CAST(NULL AS BIGINT) AS v FROM __ref(\"staged\") AS o UNION ALL \ + SELECT __polyglot_typed(NULL, 'BIGINT') AS v FROM __ref(\"staged\") AS o", + expected_column: "v BIGINT", + }, + TypedMarkerTestCase { + description: "a CTE read by name", + sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ + FROM __ref(\"staged\")) SELECT v FROM c", + expected_column: "v TEXT", + }, + TypedMarkerTestCase { + description: "an operand of a larger expression", + sql: "SELECT COALESCE(__polyglot_typed(bar(amount, 0, 10), 'VARCHAR'), 'x') AS v \ + FROM __ref(\"staged\")", + expected_column: "v TEXT", + }, + ]; + + for test_case in test_cases { + let (_, outcomes, _) = completed(started(typed_marker_request(test_case.sql))); let lines: Vec> = outcomes.iter().map(described).collect(); + let column: String = described_column_v(&lines); assert!( - column_v(&lines).starts_with(test_case.expected_column), - "{}: {}", + column.starts_with(test_case.expected_column), + "{}: {column}", test_case.description, - column_v(&lines) ); } } #[test] fn given_typed_markers_when_rerunning_from_the_cache_then_the_hit_matches_the_cold_run() { - for test_case in CASES { - let cold: CachedRun = cached_run(request(test_case.sql), NativeStore::default()); - let warm: CachedRun = cached_run(request(test_case.sql), cold.store); + let test_cases = [ + TypedMarkerCacheTestCase { + description: "an unknown body takes the asserted type", + sql: "SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a known body keeps its inferred type", + sql: "SELECT __polyglot_typed(order_id, 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a later set-operation branch types the union", + sql: "SELECT CAST(NULL AS VARCHAR) AS v FROM __ref(\"staged\") UNION ALL \ + SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v FROM __ref(\"staged\")", + expected_hits: &[true, true], + }, + TypedMarkerCacheTestCase { + description: "a CTE read by name", + sql: "WITH c AS (SELECT __polyglot_typed(bar(amount, 0, 10), 'VARCHAR') AS v \ + FROM __ref(\"staged\")) SELECT v FROM c", + expected_hits: &[true, true], + }, + ]; + + for test_case in test_cases { + let cold: CachedRun = + cached_run(typed_marker_request(test_case.sql), NativeStore::default()); + let warm: CachedRun = cached_run(typed_marker_request(test_case.sql), cold.store); assert_eq!(warm.lines, cold.lines, "{}", test_case.description); - assert!( - warm.hits.iter().all(|hit| *hit), - "{}: {:?}", - test_case.description, - warm.hits + assert_eq!( + warm.hits, test_case.expected_hits, + "{}", + test_case.description ); } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index e864a71cca..aa29f89c8d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -224,3 +224,9 @@ pub(crate) struct TypedMarkerTestCase { pub(crate) sql: &'static str, pub(crate) expected_column: &'static str, } + +pub(crate) struct TypedMarkerCacheTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_hits: &'static [bool], +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs index 3e3cbbdf0a..ad8781052c 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/compatibility_types.rs @@ -308,7 +308,6 @@ impl TypeContext<'_> { if matches!(unwrapped(inner), Expression::Null(_)) { return self.cast_type(asserted); } - // An unknown inferred type leaves the assertion to the annotation fallback. return self .expression(inner, aliases) .filter(|inferred| !inferred.to_ascii_uppercase().contains("UNKNOWN")); @@ -476,8 +475,7 @@ fn asserted_expression_type(mut expression: &Expression) -> Option<(&DataType, & } } -/// A typed-like assertion around `expression` that annotation has not resolved, past -/// parentheses and comments: the expression whose type it asserts, and what it types. +/// The like and typed expressions of an unresolved typed-like assertion wrapping `expression`. fn unresolved_like(mut expression: &Expression) -> Option<(&Expression, &Expression)> { loop { expression = match expression { diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs new file mode 100644 index 0000000000..1d22c5d16d --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs @@ -0,0 +1,40 @@ +use std::collections::HashMap; + +use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; + +use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_outputs}; + +/// The output columns and types of the one query in `sql`, typed by full type annotation. +#[must_use] +pub fn annotated_output_types( + sql: &str, + dialect: &str, + schema: &ValidationSchema, +) -> Option)>> { + let dialect: DialectType = match dialect.parse() { + Ok(dialect) => dialect, + Err(_) => return None, + }; + let mut statements: Vec = + match polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()) { + Ok(statements) => statements, + Err(_) => return None, + }; + if statements.len() != 1 { + return None; + } + let mut expression: Expression = statements.pop()?; + let mapping = polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( + schema, dialect, + ); + polyglot_sql::optimizer::annotate_types::annotate_types( + &mut expression, + Some(&mapping as &dyn polyglot_sql::schema::Schema), + Some(dialect), + ); + let mut outputs: Vec<(String, Option)> = Vec::new(); + for (name, data_type) in infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) { + outputs.push((name, data_type.filter(|value| value != NULL_TYPE))); + } + Some(outputs) +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs index 2634257679..f51bc838b0 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -1,5 +1,6 @@ pub mod analyze; pub mod analyze_project; pub mod analyze_project_compact; +pub mod annotated_output_types; pub mod output_types; pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs index 911d42b271..c7d83d7d89 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/output_types.rs @@ -2,9 +2,7 @@ use std::collections::HashMap; use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; -use crate::query_analysis::compatibility_types::{ - NULL_TYPE, infer_complete_outputs, infer_outputs, -}; +use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_complete_outputs}; /// The output columns and inferred types of the one query in `sql`; `None` if a source is untyped. #[must_use] @@ -34,36 +32,6 @@ pub fn output_types( Some(outputs) } -/// The output columns and types of the one query in `sql`, typed by full type annotation. -#[must_use] -pub fn annotated_output_types( - sql: &str, - dialect: &str, - schema: &ValidationSchema, -) -> Option)>> { - let dialect: DialectType = dialect.parse().ok()?; - let mut statements: Vec = - polyglot_sql::parse_with_options(sql, dialect, &ParseOptions::default()).ok()?; - if statements.len() != 1 { - return None; - } - let mut expression: Expression = statements.pop()?; - let mapping = polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( - schema, dialect, - ); - polyglot_sql::optimizer::annotate_types::annotate_types( - &mut expression, - Some(&mapping as &dyn polyglot_sql::schema::Schema), - Some(dialect), - ); - Some( - infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) - .into_iter() - .map(|(name, data_type)| (name, data_type.filter(|value| !is_null_type(value)))) - .collect(), - ) -} - /// Whether `value` is the placeholder inference uses for a `NULL` literal's type. fn is_null_type(value: &str) -> bool { value == NULL_TYPE diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs index f9a3154ce9..b2ed75855c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/mod.rs @@ -1,13 +1,3 @@ -pub(crate) mod arguments; -pub(crate) mod body; -pub(crate) mod cte_rename; -pub(crate) mod definition; -pub(crate) mod expansion; -pub(crate) mod header; -pub(crate) mod lexing; -pub(crate) mod literal_positions; -pub(crate) mod parameters; -pub(crate) mod placement; -pub(crate) mod probes; -pub(crate) mod returns_check; -pub(crate) mod typed_calls; +pub(crate) mod parsing; +pub(crate) mod rendering; +pub(crate) mod typing; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs similarity index 98% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs index b894b6135f..9a45512806 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs @@ -3,13 +3,13 @@ use polyglot_sql::expressions::{Identifier, Select}; use polyglot_sql::{DialectType, Expression, ExpressionWalk}; -use crate::sql_macros::_helpers::arguments::is_atomic; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::parameters::{ +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::parameters::{ ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, with_placeholders, }; -use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; +use crate::sql_macros::_helpers::rendering::arguments::is_atomic; +use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs similarity index 96% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs index da57a534db..c401284890 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs @@ -1,13 +1,15 @@ //! A SQL macro file parsed into its definition, then checked on its own. -use crate::sql_macros::_helpers::arguments::{ShapeContext, argument_shape, check_shape}; -use crate::sql_macros::_helpers::header::{Header, HeaderParser}; -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs}; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, line_column, tokens}; -use crate::sql_macros::_helpers::literal_positions::{ +use crate::sql_macros::_helpers::parsing::header::{Header, HeaderParser}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; -use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::rendering::arguments::{ + ShapeContext, argument_shape, check_shape, +}; use crate::sql_macros::constants::{ BARE_MACRO_CALL_CODE, BODY_REFERENCE_CODE, INVALID_DEFINITION_CODE, MIXED_KIND_CODE, PARAMETER_REFERENCE_CODE, REFERENCE_FUNCTIONS, RESERVED_PARAMETER_NAMES, SELECT_KEYWORD, @@ -355,7 +357,9 @@ fn check_references( let referenced: String = scanned .get(index + 2) .filter(|argument| matches!(argument.kind, TokenKind::Quoted | TokenKind::Text)) - .map(|argument| crate::sql_macros::_helpers::header::unquoted(argument.text(body))) + .map(|argument| { + crate::sql_macros::_helpers::parsing::header::unquoted(argument.text(body)) + }) .unwrap_or_default(); let kind: &str = match function { "__source" => "source", diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs similarity index 99% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs index 2704b94e98..3ed9989d10 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/header.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs @@ -1,6 +1,8 @@ //! The `MACRO (...)` header of a SQL macro file: description, typed arguments and result type. -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{ + Token, TokenKind, closing_token, line_column, tokens, +}; use crate::sql_macros::constants::{ ANY_KEYWORD, ARGUMENTS_KEY, COLUMN_KEYWORD, COMMA, CONST_KEYWORD, DEFAULT_KEYWORD, DESCRIPTION_KEY, ENUM_KEYWORD, HEADER_KEYWORD, INVALID_DEFINITION_CODE, LIST_KEYWORD, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs similarity index 100% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/lexing.rs diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs similarity index 98% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs index 6d24941fdc..839e030ed8 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/literal_positions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs @@ -1,7 +1,7 @@ //! Parameters a SQL macro body uses where SQL requires a literal, such as `LIMIT @n`. -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token}; -use crate::sql_macros::_helpers::parameters::ParameterReference; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token}; +use crate::sql_macros::_helpers::parsing::parameters::ParameterReference; use crate::sql_macros::constants::{ LITERAL_REQUIRED_CODE, ROUND_FUNCTION, ROUND_PRECISION_ARGUMENT, }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs new file mode 100644 index 0000000000..7e98da38eb --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod body; +pub(crate) mod definition; +pub(crate) mod header; +pub(crate) mod lexing; +pub(crate) mod literal_positions; +pub(crate) mod parameters; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs similarity index 97% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs index 05039f6b00..d825103bb4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/parameters.rs @@ -2,7 +2,7 @@ use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; -use crate::sql_macros::_helpers::lexing::{DECLARATION_WORDS, Token, TokenKind}; +use crate::sql_macros::_helpers::parsing::lexing::{DECLARATION_WORDS, Token, TokenKind}; const PLACEHOLDER_PREFIX: &str = "__sqlbuild_parameter__"; /// Words that may follow a table reference without being its alias. diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs similarity index 98% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs index b799e90eec..4a32f7256f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs @@ -1,8 +1,8 @@ //! A call's argument list: split, bound to parameters and checked against their types. -use crate::sql_macros::_helpers::header::{split_top_level, unquoted}; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token, tokens}; -use crate::sql_macros::_helpers::returns_check::adapter_dialect; +use crate::sql_macros::_helpers::parsing::header::{split_top_level, unquoted}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; use crate::sql_macros::constants::{ ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, }; @@ -239,7 +239,7 @@ pub(crate) fn argument_shape(text: &str) -> ArgumentShape { } fn unquote(text: &str) -> String { - crate::sql_macros::_helpers::header::unquoted(text) + crate::sql_macros::_helpers::parsing::header::unquoted(text) } /// The type family of a SQL literal argument. diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs similarity index 95% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs index 1e1a082e1c..29118b660e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/cte_rename.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs @@ -3,11 +3,11 @@ use polyglot_sql::expressions::Identifier; use polyglot_sql::{DialectType, Expression, ExpressionWalk}; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::parameters::{ +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::parameters::{ ParameterReference, parameter_references, placeholder, with_placeholders, }; -use crate::sql_macros::_helpers::typed_calls::spliced; +use crate::sql_macros::_helpers::typing::typed_calls::spliced; use crate::sql_macros::constants::{ARGUMENT_CODE, CTE_ARGUMENT, CTE_NAME_SEPARATOR}; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ @@ -117,13 +117,14 @@ pub(crate) fn renamed_ctes(body: &ExpandedText, name: &str) -> Result = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); - let aliases: Vec = statement - .dfs() - .filter_map(|node| match node { - Expression::Table(table) => table.alias.as_ref().map(|alias| alias.name.clone()), - _ => None, - }) - .collect(); + let mut aliases: Vec = Vec::new(); + for node in statement.dfs() { + if let Expression::Table(table) = node + && let Some(alias) = &table.alias + { + aliases.push(alias.name.clone()); + } + } let mut identifiers: Vec<&Identifier> = with.ctes.iter().map(|cte| &cte.alias).collect(); for node in statement.dfs() { match node { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs similarity index 96% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs index a029bdd94d..77c4d9c290 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs @@ -3,21 +3,21 @@ use std::sync::Arc; use crate::compiler::main::function_type_validation::function_type_error; -use crate::sql_macros::_helpers::arguments::{ - ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, -}; -use crate::sql_macros::_helpers::body::{expand_body, with_line_comment_closed}; -use crate::sql_macros::_helpers::cte_rename::{renamed_ctes, take_cte_argument}; -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs, outer_sql_calls}; -use crate::sql_macros::_helpers::lexing::{Token, line_column, tokens}; -use crate::sql_macros::_helpers::literal_positions::{ +use crate::sql_macros::_helpers::parsing::body::{expand_body, with_line_comment_closed}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs, outer_sql_calls}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, line_column, tokens}; +use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; -use crate::sql_macros::_helpers::parameters::{ParameterReference, parameter_references}; -use crate::sql_macros::_helpers::placement::top_level_parts; -use crate::sql_macros::_helpers::placement::{CallSite, cte_names, select_names}; -use crate::sql_macros::_helpers::returns_check::{check_body_returns, check_body_types}; -use crate::sql_macros::_helpers::typed_calls::{shifted, spliced}; +use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::rendering::arguments::{ + ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, +}; +use crate::sql_macros::_helpers::rendering::cte_rename::{renamed_ctes, take_cte_argument}; +use crate::sql_macros::_helpers::rendering::placement::top_level_parts; +use crate::sql_macros::_helpers::rendering::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::typing::returns_check::{check_body_returns, check_body_types}; +use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ ARGUMENT_CODE, CTE_ARGUMENT, CTE_NOUN, CYCLE_CODE, INVALID_DEFINITION_CODE, MISPLACED_CALL_CODE, NAME_CLASH_CODE, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs new file mode 100644 index 0000000000..9b07958fe7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod arguments; +pub(crate) mod cte_rename; +pub(crate) mod expansion; +pub(crate) mod placement; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs similarity index 98% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs index 209ddd5283..d365663e92 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/placement.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/placement.rs @@ -1,7 +1,7 @@ //! Where a call stands in its caller: its query part, separators and the caller's names. -use crate::sql_macros::_helpers::header::unquoted; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, closing_token}; +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token}; use crate::sql_macros::constants::{AT_SIGN, COMMA, SELECT_KEYWORD, WITH_KEYWORD}; use crate::sql_macros::models::{CallPosition, CallScope}; @@ -389,7 +389,7 @@ pub(crate) fn cte_names(sql: &str, tokens: &[Token], with_index: usize) -> Vec<( if !matches!(token.kind, TokenKind::Word | TokenKind::Quoted) { break; } - let name: String = crate::sql_macros::_helpers::header::unquoted(token.text(sql)); + let name: String = crate::sql_macros::_helpers::parsing::header::unquoted(token.text(sql)); let mut cursor: usize = index + 1; if tokens.get(cursor).is_some_and(|after| after.is(sql, "(")) { match closing_token(sql, tokens, cursor) { @@ -488,7 +488,7 @@ fn item_name(sql: &str, item: &[Token]) -> Option<(String, usize)> { }; named.then(|| { ( - crate::sql_macros::_helpers::header::unquoted(last.text(sql)), + crate::sql_macros::_helpers::parsing::header::unquoted(last.text(sql)), last.start, ) }) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs new file mode 100644 index 0000000000..7c872088b8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod probes; +pub(crate) mod returns_check; +pub(crate) mod typed_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs similarity index 98% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs index 3740d03294..583231d189 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/probes.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs @@ -4,10 +4,10 @@ use polyglot_sql::ValidationSchema; use serde_json::{Value, json}; use crate::query_analysis::main::output_types::output_types; -use crate::sql_macros::_helpers::arguments::{fits, type_family}; -use crate::sql_macros::_helpers::header::unquoted; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, tokens}; -use crate::sql_macros::_helpers::placement::{QueryParts, top_level_parts}; +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::_helpers::rendering::arguments::{fits, type_family}; +use crate::sql_macros::_helpers::rendering::placement::{QueryParts, top_level_parts}; use crate::sql_macros::constants::{ ALIAS_PRECEDING_KEYWORDS, FROM_KEYWORD, INFERRED_TEXT_TYPE, PROBE_COLUMN_PREFIX, PROBE_TAIL_ENDS, STAR_PROJECTION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs similarity index 75% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index b22da7d5c0..1133098f65 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -6,11 +6,10 @@ use std::sync::{Mutex, OnceLock}; use polyglot_sql::{DialectType, SchemaValidationOptions, ValidationSchema, validate_with_schema}; use serde_json::{Value, json}; -use crate::contracts::_helpers::type_comparison::{TypeComparer, TypeComparison}; -use crate::query_analysis::main::output_types::annotated_output_types; -use crate::sql_macros::_helpers::arguments::type_family; -use crate::sql_macros::_helpers::body::expand_body; -use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::query_analysis::main::annotated_output_types::annotated_output_types; +use crate::sql_macros::_helpers::parsing::body::expand_body; +use crate::sql_macros::_helpers::rendering::arguments::type_family; +use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; use crate::sql_macros::constants::{ BODY_TYPE_CODE, INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, TYPE_ERROR_CODES, TYPE_WARNING_CODES, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, @@ -20,6 +19,8 @@ use crate::sql_macros::models::{ ArgumentShape, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, SqlMacroForm, SqlMacroLocation, TypeFamily, }; +use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::TypeNormalization; /// Each body check's verdict, by the probe it ran, so a macro is typed once per process. static VERDICTS: OnceLock>>> = OnceLock::new(); @@ -33,8 +34,7 @@ pub(crate) fn adapter_dialect(adapter: &str) -> &str { } } -/// The type a parameter's stand-in column has, or `None` for an untyped stand-in; an `ANY` -/// family takes its widest member when `families` is set. +/// A parameter's stand-in column type, with `ANY` families widened only when `families` is set. fn stand_in_type(kind: &ParameterKind, families: bool) -> Option<&str> { match kind { ParameterKind::Expression { data_type } | ParameterKind::Constant { data_type } => { @@ -56,8 +56,7 @@ fn stand_in_column(index: usize) -> String { format!("p{index}") } -/// The query selecting the body with stand-ins for its arguments, and the stand-ins' relation, -/// unless the body reads relations or columns. +/// The body selected over stand-in arguments, and their relation, unless it reads relations. fn body_probe( definition: &SqlMacroDefinition, body: &ExpandedText, @@ -88,7 +87,10 @@ fn body_probe( calls: Vec::new(), }); } - let expansion = expand_body(definition, body, &arguments).ok()?; + let expansion = match expand_body(definition, body, &arguments) { + Ok(expansion) => expansion, + Err(_) => return None, + }; let (typed, _) = typed_sql(&expansion.sql, &expansion.calls); let probe: String = format!("SELECT ({typed}) AS b FROM {STAND_IN_RELATION}"); let table: Value = json!({"name": STAND_IN_RELATION, "columns": columns}); @@ -97,15 +99,16 @@ fn body_probe( /// The body's inferred type, when inference knows it completely. fn inferred_body_type(probe: &str, table: Value, dialect: &str) -> Option { - let schema: ValidationSchema = serde_json::from_value(json!({ "tables": [table] })).ok()?; + let Ok(schema) = serde_json::from_value::(json!({ "tables": [table] })) + else { + return None; + }; let outputs = annotated_output_types(probe, dialect, &schema)?; let (_, data_type) = outputs.into_iter().next()?; data_type.filter(|inferred| !inferred.to_ascii_uppercase().contains(UNKNOWN_TYPE_HINT)) } -/// Whether a body of type `inferred` disagrees with `returns`: in another type family, or, for -/// types outside the scalar families, of another type. Within one family the call's own -/// inference decides, since stand-ins lose what literal arguments tell it. +/// Whether body type `inferred` is outside the family of `returns`, or another non-family type. fn disagrees(returns: &str, inferred: &str, adapter: &str) -> bool { let (declared, found) = ( type_family(returns, adapter), @@ -114,10 +117,37 @@ fn disagrees(returns: &str, inferred: &str, adapter: &str) -> bool { if declared != TypeFamily::Unknown && found != TypeFamily::Unknown { return declared != found; } - matches!( - TypeComparer::new(adapter_dialect(adapter)).types_equal(returns, inferred), - Ok(TypeComparison::Different) - ) + let dialect: &str = adapter_dialect(adapter); + match ( + normalize_type(returns, dialect), + normalize_type(inferred, dialect), + ) { + ( + Ok(TypeNormalization { + normalized: declared, + .. + }), + Ok(TypeNormalization { + normalized: found, .. + }), + ) => declared != found, + _ => false, + } +} + +/// The verdict cached under `key`, or `compute`'s verdict, cached for the rest of the process. +fn remembered(key: String, compute: impl FnOnce() -> Option) -> Option { + let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); + let mut verdicts = match verdicts.lock() { + Ok(verdicts) => verdicts, + Err(poisoned) => poisoned.into_inner(), + }; + if let Some(known) = verdicts.get(&key) { + return known.clone(); + } + let verdict: Option = compute(); + verdicts.insert(key, verdict.clone()); + verdict } /// Rejects an expression macro whose body's known type is not its declared `returns`. @@ -137,21 +167,7 @@ pub(crate) fn check_body_returns( }; let dialect: &str = adapter_dialect(adapter); let key: String = format!("returns\n{dialect}\n{table}\n{probe}"); - let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); - let known: Option> = verdicts - .lock() - .ok() - .and_then(|verdicts| verdicts.get(&key).cloned()); - let inferred: Option = match known { - Some(inferred) => inferred, - None => { - let inferred: Option = inferred_body_type(&probe, table, dialect); - if let Ok(mut verdicts) = verdicts.lock() { - verdicts.insert(key, inferred.clone()); - } - inferred - } - }; + let inferred: Option = remembered(key, || inferred_body_type(&probe, table, dialect)); let Some(inferred) = inferred else { return Ok(()); }; @@ -178,8 +194,14 @@ pub(crate) fn check_body_returns( /// The first type error the body's probe shows, as polyglot reports it. fn body_type_error(probe: &str, table: Value, dialect: &str) -> Option { - let schema: ValidationSchema = serde_json::from_value(json!({ "tables": [table] })).ok()?; - let dialect_type: DialectType = dialect.parse().ok()?; + let Ok(schema) = serde_json::from_value::(json!({ "tables": [table] })) + else { + return None; + }; + let dialect_type: DialectType = match dialect.parse() { + Ok(dialect_type) => dialect_type, + Err(_) => return None, + }; let options: SchemaValidationOptions = SchemaValidationOptions { check_types: true, strict: Some(false), @@ -194,11 +216,9 @@ fn body_type_error(probe: &str, table: Value, dialect: &str) -> Option { /// Whether a polyglot validation code reports an operand or argument of the wrong type. fn is_type_code(code: &str) -> bool { - let Some(number) = code - .get(1..) - .and_then(|digits| digits.parse::().ok()) - else { - return false; + let number: usize = match code.get(1..).map(str::parse::) { + Some(Ok(number)) => number, + _ => return false, }; (code.starts_with('E') && (TYPE_ERROR_CODES.0..=TYPE_ERROR_CODES.1).contains(&number)) || (code.starts_with('W') @@ -229,21 +249,7 @@ pub(crate) fn check_body_types( }; let dialect: &str = adapter_dialect(adapter); let key: String = format!("types\n{dialect}\n{table}\n{probe}"); - let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); - let known: Option> = verdicts - .lock() - .ok() - .and_then(|verdicts| verdicts.get(&key).cloned()); - let found: Option = match known { - Some(found) => found, - None => { - let found: Option = body_type_error(&probe, table, dialect); - if let Ok(mut verdicts) = verdicts.lock() { - verdicts.insert(key, found.clone()); - } - found - } - }; + let found: Option = remembered(key, || body_type_error(&probe, table, dialect)); let Some(found) = found else { return Ok(()); }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs similarity index 96% rename from crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs rename to crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs index dee823347a..1bdf1bf058 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -3,7 +3,7 @@ use polyglot_sql::expressions::{TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION}; use crate::compiled_project::models::SplicedSpan; -use crate::sql_macros::_helpers::body::with_line_comment_closed; +use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `calls` moved `offset` bytes later. @@ -28,7 +28,10 @@ fn spliced_position(position: usize, splices: &[TextSplice]) -> Option { break; } } - usize::try_from(position as isize + moved).ok() + let Ok(moved_position) = usize::try_from(position as isize + moved) else { + return None; + }; + Some(moved_position) } /// `calls` through `splices`, sorted and non-overlapping; a call a splice cuts into is dropped. diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 2f5bd29fb7..7c0e88879a 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -109,7 +109,8 @@ pub(crate) const PROBE_TAIL_ENDS: [&str; 11] = [ pub(crate) const INFERRED_TEXT_TYPE: &str = "TEXT"; pub(crate) const VARCHAR_TYPE: &str = "VARCHAR"; pub(crate) const FROM_KEYWORD: &str = "FROM"; -/// The projection name of an unexpanded star in a probe's analysis. +/// Words that end a select item rather than alias it. +pub(crate) const ITEM_ENDS: [&str; 4] = ["FROM", "WHERE", "UNION", "ORDER"]; /// The relation whose columns stand in for a macro's arguments while its body is typed. pub(crate) const STAND_IN_RELATION: &str = "__sqlbuild_macro_arguments"; /// The polyglot validation error codes, by number, of operands and arguments of a wrong type. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs index 32277f5163..5f36056974 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs}; +use crate::sql_macros::_helpers::parsing::lexing::{AtConstruct, at_constructs}; use crate::sql_macros::constants::NON_ASCII_BYTE; use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs index 3fad5582e7..5836d87b82 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/byte_text_splices.rs @@ -1,5 +1,5 @@ use crate::compiled_project::models::SplicedSpan; -use crate::sql_macros::_helpers::typed_calls::byte_splices; +use crate::sql_macros::_helpers::typing::typed_calls::byte_splices; use crate::sql_macros::models::TextSplice; /// Character-offset `spans` from `source` to `output` as byte splices. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs index 4deecba367..b6acd14352 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/check_sql_macro_arguments.rs @@ -1,4 +1,6 @@ -use crate::sql_macros::_helpers::probes::{ProbePlan, analysed_probes, check_findings, probe_plan}; +use crate::sql_macros::_helpers::typing::probes::{ + ProbePlan, analysed_probes, check_findings, probe_plan, +}; use crate::sql_macros::models::SqlMacroFinding; use crate::sql_macros::types::{ArgumentCheckInput, ProbeColumns}; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs index c1216cf98e..a7139e3c65 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/expand_sql_macro_call.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; use crate::sql_macros::constants::INVALID_DEFINITION_CODE; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; @@ -18,5 +18,10 @@ pub fn expand_sql_macro_call( "SQL has an unclosed quote or comment", ) })?; - crate::sql_macros::_helpers::expansion::expand_call(host, context, (&scanned, inserted), call) + crate::sql_macros::_helpers::rendering::expansion::expand_call( + host, + context, + (&scanned, inserted), + call, + ) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs index b84e2ab843..caff6c30c2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/moved_typed_calls.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::typed_calls::spliced; +use crate::sql_macros::_helpers::typing::typed_calls::spliced; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `calls` moved by byte `splices`, sorted and non-overlapping; a call a splice cuts is dropped. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs b/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs index cd9d71100a..190aa37629 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/parse_sql_macro.rs @@ -1,4 +1,4 @@ -use crate::sql_macros::_helpers::definition::parsed_definition; +use crate::sql_macros::_helpers::parsing::definition::parsed_definition; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::SqlMacroDefinition; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs b/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs index 5f02404a7e..8d20f12559 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/render_inline_collection.rs @@ -1,5 +1,5 @@ -use crate::sql_macros::_helpers::arguments::{collection_items, collection_problem}; -use crate::sql_macros::_helpers::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::rendering::arguments::{collection_items, collection_problem}; use crate::sql_macros::constants::ARGUMENT_CODE; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::TypeFamily; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs index c32493b136..14dfa9fd39 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/spliced_typed_calls.rs @@ -1,5 +1,5 @@ use crate::compiled_project::models::SplicedSpan; -use crate::sql_macros::_helpers::typed_calls::{byte_splices, spliced}; +use crate::sql_macros::_helpers::typing::typed_calls::{byte_splices, spliced}; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `calls` in `source` moved to `output` by character-offset `spans`; a call a span cuts is dropped. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs index 62db814bb3..91e989505d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs @@ -1,7 +1,7 @@ //! The form of a SQL macro body, as its definition reads it. -use crate::sql_macros::_helpers::definition::body_form; -use crate::sql_macros::_helpers::lexing::tokens; +use crate::sql_macros::_helpers::parsing::definition::body_form; +use crate::sql_macros::_helpers::parsing::lexing::tokens; use crate::sql_macros::models::SqlMacroForm; /// Whether `body` is a CTE list, a column list or an expression, from its first token. diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs index 3af9c1578f..aa5ad4a182 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_calls.rs @@ -6,5 +6,5 @@ pub fn sql_macro_calls(text: &str) -> Result, SqlMacroError> { if !text.contains(crate::sql_macros::constants::SQL_CALL_PREFIX) { return Ok(Vec::new()); } - crate::sql_macros::_helpers::expansion::sql_macro_calls(text) + crate::sql_macros::_helpers::rendering::expansion::sql_macro_calls(text) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs index 419d4fcaff..58a8a05f64 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs @@ -1,7 +1,7 @@ //! The byte spans of a SQL macro body's `@name` parameter references, as expansion reads them. -use crate::sql_macros::_helpers::lexing::tokens; -use crate::sql_macros::_helpers::parameters::parameter_references; +use crate::sql_macros::_helpers::parsing::lexing::tokens; +use crate::sql_macros::_helpers::parsing::parameters::parameter_references; /// The `start..end` byte span of each `@name` in `body` that expansion replaces by an argument. #[must_use] diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs index 1410c04f87..854227c5e2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs @@ -1,9 +1,8 @@ use crate::semantic_validation::models::Expansion; -use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; use crate::sql_macros::models::TypedCall; -/// `sql` with its typed calls marked, and the pass mapping the marked text's character offsets -/// back to `sql`, or `None` when no call is marked. +/// `sql` with its typed calls marked and the pass mapping offsets back, if any call is marked. #[must_use] pub fn typed_analysis_pass(sql: &str, calls: &[TypedCall]) -> Option<(String, Vec)> { let (typed, insertions) = typed_sql(sql, calls); diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs index 2550414f76..0740dc6155 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs @@ -1,8 +1,7 @@ -use crate::sql_macros::_helpers::typed_calls::typed_sql; +use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; use crate::sql_macros::models::{TextSplice, TypedCall}; -/// `sql` with each call that declares `returns` wrapped in the analysis-only typed marker, and -/// each marker insertion as an empty source range of `sql`. +/// `sql` with its typed calls wrapped in analysis-only markers, and each marker insertion. #[must_use] pub fn typed_analysis_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { typed_sql(sql, calls) diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs index 9979907369..cc4c9115da 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/untyped_call_outputs.rs @@ -1,12 +1,9 @@ -use crate::sql_macros::_helpers::header::unquoted; -use crate::sql_macros::_helpers::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, tokens}; +use crate::sql_macros::constants::ITEM_ENDS; use crate::sql_macros::models::TypedCall; -/// Words that end a select item rather than alias it. -const ITEM_ENDS: [&str; 4] = ["FROM", "WHERE", "UNION", "ORDER"]; - -/// Each named select item of `sql` that is exactly one call to a macro without `returns`, as -/// `(output name, macro name)`. +/// The `(output name, macro name)` of each select item that is one untyped macro call. #[must_use] pub fn untyped_call_outputs(sql: &str, calls: &[TypedCall]) -> Vec<(String, String)> { let untyped: Vec<&TypedCall> = calls @@ -40,8 +37,7 @@ pub fn untyped_call_outputs(sql: &str, calls: &[TypedCall]) -> Vec<(String, Stri after }; let Some(name) = scanned.get(alias).filter(|token| { - matches!(token.kind, TokenKind::Word | TokenKind::Quoted) - && !ITEM_ENDS.iter().any(|word| token.is_word(sql, word)) + matches!(token.kind, TokenKind::Word | TokenKind::Quoted) && !ends_item(token, sql) }) else { continue; }; @@ -54,3 +50,8 @@ pub fn untyped_call_outputs(sql: &str, calls: &[TypedCall]) -> Vec<(String, Stri } outputs } + +/// Whether `token` is a word that ends a select item rather than aliases it. +fn ends_item(token: &Token, sql: &str) -> bool { + ITEM_ENDS.iter().any(|word| token.is_word(sql, word)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs index 24fc2bf9f8..9669935131 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/with_list_sql_macro_calls.rs @@ -1,5 +1,5 @@ -use crate::sql_macros::_helpers::lexing::{Token, tokens}; -use crate::sql_macros::_helpers::placement::CallSite; +use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; +use crate::sql_macros::_helpers::rendering::placement::CallSite; use crate::sql_macros::main::sql_macro_calls::sql_macro_calls; use crate::sql_macros::models::{CallPosition, SqlMacroCall}; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index 0205c590c0..fafd5775b0 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use crate::sql_macros::_helpers::expansion::expand_text; +use crate::sql_macros::_helpers::rendering::expansion::expand_text; use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; use crate::sql_macros::models::{ - ExpandedText, RenderedDeclarations, SqlMacroDefinition, TextContext, TextSplice, + ExpandedText, RenderedDeclarations, SqlMacroDefinition, TextContext, TextSplice, TypedCall, }; use crate::sql_macros::tests::test_types::{AsciiArguments, FakeHost}; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -154,3 +154,35 @@ impl sqlbuild_render::macro_arguments::types::ArgumentHost for AsciiArguments { None } } + +/// Each call as `macro returns: text`. +pub(super) fn shown_typed_calls(sql: &str, calls: &[TypedCall]) -> Vec { + calls + .iter() + .map(|call| { + format!( + "{} {}: {}", + call.macro_name, + call.returns.as_deref().unwrap_or("-"), + &sql[call.start..call.end] + ) + }) + .collect() +} + +/// An untyped `shout` call of each text in `sql`. +pub(super) fn untyped_calls(sql: &str, texts: &[&str]) -> Vec { + texts + .iter() + .map(|text| { + let start: usize = sql.find(text).unwrap_or_default(); + TypedCall { + returns_like: None, + start, + end: start + text.len(), + macro_name: "shout".to_owned(), + returns: None, + } + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 98e15a0ad0..1ea1261529 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -1,8 +1,11 @@ use crate::sql_macros::main::typed_analysis_sql::typed_analysis_sql; use crate::sql_macros::main::untyped_call_outputs::untyped_call_outputs; use crate::sql_macros::models::{ExpandedText, TypedCall}; -use crate::sql_macros::tests::helpers::expanded_text_for; -use crate::sql_macros::tests::test_types::{TypedAnalysisTestCase, TypedCallTestCase}; +use crate::sql_macros::tests::helpers::{expanded_text_for, shown_typed_calls, untyped_calls}; +use crate::sql_macros::tests::test_types::{ + ExpandedAnalysisTestCase, ReservedNameTestCase, TypedAnalysisTestCase, TypedCallTestCase, + UntypedOutputsTestCase, +}; const SHOUT: (&str, &str) = ( "shout", @@ -30,20 +33,11 @@ const PLACES: (&str, &str) = ( "MACRO (description \"Places\", arguments (value BIGINT), returns BIGINT);\n\n\ @value + @const(\"price_places\")\n", ); - -fn shown(sql: &str, calls: &[TypedCall]) -> Vec { - calls - .iter() - .map(|call| { - format!( - "{} {}: {}", - call.macro_name, - call.returns.as_deref().unwrap_or("-"), - &sql[call.start..call.end] - ) - }) - .collect() -} +const LIKE_VALUE: (&str, &str) = ( + "like_value", + "MACRO (description \"Like\", arguments (value ANY, fallback ANY DEFAULT NULL), \ + returns TYPE OF value);\n\nCOALESCE(@value, @fallback)\n", +); #[test] fn given_expression_macro_calls_when_expanding_then_each_expansion_is_recorded_where_it_lands() { @@ -112,7 +106,7 @@ fn given_expression_macro_calls_when_expanding_then_each_expansion_is_recorded_w let expanded = expanded_text_for("duckdb", test_case.macros, test_case.sql) .unwrap_or_else(|error| panic!("{}: {error:?}", test_case.description)); assert_eq!( - shown(&expanded.sql, &expanded.calls), + shown_typed_calls(&expanded.sql, &expanded.calls), test_case.expected_calls, "{}: {}", test_case.description, @@ -191,62 +185,72 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns #[test] fn given_the_reserved_analysis_function_when_comparing_then_authored_sql_rejects_the_parsed_name() { - assert_eq!( - sqlbuild_attachments::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION, - polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION - ); + let test_cases = [ReservedNameTestCase { + description: "the typed marker", + authored_name: + sqlbuild_attachments::cursor_intrinsics::constants::RESERVED_ANALYSIS_FUNCTION, + expected_parsed_name: polyglot_sql::expressions::TYPED_EXPRESSION_FUNCTION, + }]; + + for test_case in test_cases { + assert_eq!( + test_case.authored_name, test_case.expected_parsed_name, + "{}", + test_case.description + ); + } } #[test] fn given_calls_without_returns_when_listing_outputs_then_only_whole_named_items_are_named() { - let untyped = |start: usize, end: usize| TypedCall { - returns_like: None, - start, - end, - macro_name: "shout".to_owned(), - returns: None, - }; - let sql: &str = - "SELECT UPPER(a) AS loud, UPPER(b) quiet, UPPER(c) || 'x' AS joined, UPPER(d) FROM t"; - let calls: Vec = ["UPPER(a)", "UPPER(b)", "UPPER(c)", "UPPER(d)"] - .iter() - .map(|text| { - let start: usize = sql.find(text).unwrap_or_default(); - untyped(start, start + text.len()) - }) - .collect(); + let test_cases = [ + UntypedOutputsTestCase { + description: "aliased whole calls are named, operands and unnamed items are not", + sql: "SELECT UPPER(a) AS loud, UPPER(b) quiet, UPPER(c) || 'x' AS joined, UPPER(d) FROM t", + call_texts: &["UPPER(a)", "UPPER(b)", "UPPER(c)", "UPPER(d)"], + expected_outputs: &[("loud", "shout"), ("quiet", "shout")], + }, + UntypedOutputsTestCase { + description: "a keyword after a call is not its alias", + sql: "SELECT id, UPPER(a) FROM t", + call_texts: &["UPPER(a)"], + expected_outputs: &[], + }, + ]; - let outputs: Vec<(String, String)> = untyped_call_outputs(sql, &calls); + for test_case in test_cases { + let calls: Vec = untyped_calls(test_case.sql, test_case.call_texts); - assert_eq!( - outputs, - vec![ - ("loud".to_owned(), "shout".to_owned()), - ("quiet".to_owned(), "shout".to_owned()), - ] - ); -} + let outputs: Vec<(String, String)> = untyped_call_outputs(test_case.sql, &calls); -const LIKE_VALUE: (&str, &str) = ( - "like_value", - "MACRO (description \"Like\", arguments (value ANY, fallback ANY DEFAULT NULL), \ - returns TYPE OF value);\n\nCOALESCE(@value, @fallback)\n", -); + let named: Vec<(&str, &str)> = outputs + .iter() + .map(|(output, name)| (output.as_str(), name.as_str())) + .collect(); + assert_eq!( + named, test_case.expected_outputs, + "{}", + test_case.description + ); + } +} #[test] fn given_a_returns_type_of_call_when_building_the_analysis_text_then_it_asserts_its_argument() { - let expanded: ExpandedText = expanded_text_for( - "duckdb", - &[LIKE_VALUE], - "SELECT @sql.like_value(o.created_at + 1) AS v FROM orders AS o", - ) - .expect("expands"); + let test_cases = [ExpandedAnalysisTestCase { + description: "the call is typed like its argument", + macros: &[LIKE_VALUE], + sql: "SELECT @sql.like_value(o.created_at + 1) AS v FROM orders AS o", + expected_sql: "SELECT __polyglot_typed_like(COALESCE((o.created_at + 1), NULL), \ + o.created_at + 1) AS v FROM orders AS o", + }]; - let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls); + for test_case in test_cases { + let expanded: ExpandedText = + expanded_text_for("duckdb", test_case.macros, test_case.sql).expect("expands"); + + let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls); - assert_eq!( - typed, - "SELECT __polyglot_typed_like(COALESCE((o.created_at + 1), NULL), o.created_at + 1) \ - AS v FROM orders AS o" - ); + assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); + } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 06bf27cb5c..a3164977ff 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -91,3 +91,26 @@ pub(super) struct TypedAnalysisTestCase { pub(super) calls: &'static [(usize, usize, Option<&'static str>)], pub(super) expected_sql: &'static str, } + +/// Two names that must stay equal. +pub(super) struct ReservedNameTestCase { + pub(super) description: &'static str, + pub(super) authored_name: &'static str, + pub(super) expected_parsed_name: &'static str, +} + +/// A query and the texts of its untyped calls, with the outputs each whole call names. +pub(super) struct UntypedOutputsTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) call_texts: &'static [&'static str], + pub(super) expected_outputs: &'static [(&'static str, &'static str)], +} + +/// A query calling `macros`, with the analysis text its expansion marks. +pub(super) struct ExpandedAnalysisTestCase { + pub(super) description: &'static str, + pub(super) macros: &'static [(&'static str, &'static str)], + pub(super) sql: &'static str, + pub(super) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs index 7309ba181d..cdb7d766da 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod replacement; pub(crate) mod reserved; pub(crate) mod scan; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs new file mode 100644 index 0000000000..ba51a73d01 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/replacement.rs @@ -0,0 +1,62 @@ +//! Cursor intrinsic calls replaced with the byte span of each replacement. + +use crate::cursor_intrinsics::_helpers::scan::{ + CallScan, ScanStep, call_scan, contains_intrinsic_name, step, +}; +use crate::cursor_intrinsics::models::IntrinsicSpan; +use crate::sql_lexing::main::unclosed_message::unclosed_message; + +/// `sql` with each cursor intrinsic call replaced and each call's byte span, or the call's error. +pub(crate) fn replaced_with_spans( + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, Vec), String> { + if !contains_intrinsic_name(sql) { + return Ok((sql.to_owned(), Vec::new())); + } + let mut replaced: String = String::with_capacity(sql.len()); + let mut last: usize = 0; + let mut spans: Vec = Vec::new(); + let mut index: usize = 0; + while index < sql.len() { + let (name, end): (&str, usize) = match step(sql, index) { + ScanStep::Next(next) => { + index = next; + continue; + } + ScanStep::Unclosed(construct) => return Err(unclosed_message(context, construct)), + ScanStep::Intrinsic { name, end } => (name, end), + }; + let close: usize = match call_scan(sql, end) { + CallScan::Called(close) => close, + CallScan::NotCalled => { + return Err(format!("{context} intrinsic {name} must be called with ()")); + } + CallScan::Arguments => { + return Err(format!( + "{context} intrinsic {name} does not accept arguments" + )); + } + CallScan::Unclosed(construct) => { + return Err(unclosed_message( + &format!("{context} cursor intrinsic"), + construct, + )); + } + }; + replaced.push_str(&sql[last..index]); + let output_start: usize = replaced.len(); + replaced.push_str(&replacement(name)); + spans.push(IntrinsicSpan { + source_start: index, + source_end: close + 1, + output_start, + output_end: replaced.len(), + }); + last = close + 1; + index = close + 1; + } + replaced.push_str(&sql[last..]); + Ok((replaced, spans)) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs index ac124fa97a..d23f7da39e 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs @@ -1,3 +1,4 @@ pub mod intrinsic_free; pub mod replace_intrinsics; +pub mod replace_intrinsics_with_spans; pub mod validated_model_intrinsics; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs index 704b7d4e16..f90dcd8c3f 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs @@ -1,9 +1,6 @@ //! `_transform_cursor_intrinsics`: replace every intrinsic call, or raise its error. -use crate::cursor_intrinsics::_helpers::scan::{ - CallScan, ScanStep, call_scan, contains_intrinsic_name, step, -}; -use crate::sql_lexing::main::unclosed_message::unclosed_message; +use crate::cursor_intrinsics::_helpers::replacement::replaced_with_spans; /// `sql` with each cursor intrinsic call replaced and whether any was, or the malformed-call error. pub fn replace_intrinsics( @@ -11,57 +8,6 @@ pub fn replace_intrinsics( context: &str, replacement: impl Fn(&str) -> String, ) -> Result<(String, bool), String> { - replace_intrinsics_with_spans(sql, context, replacement) + replaced_with_spans(sql, context, replacement) .map(|(replaced, spans)| (replaced, !spans.is_empty())) } - -/// `sql` with each cursor intrinsic call replaced, and each call's byte range -/// `(source start, source end, output start, output end)`, or the malformed-call error. -pub fn replace_intrinsics_with_spans( - sql: &str, - context: &str, - replacement: impl Fn(&str) -> String, -) -> Result<(String, Vec<(usize, usize, usize, usize)>), String> { - if !contains_intrinsic_name(sql) { - return Ok((sql.to_owned(), Vec::new())); - } - let mut replaced: String = String::with_capacity(sql.len()); - let mut last: usize = 0; - let mut spans: Vec<(usize, usize, usize, usize)> = Vec::new(); - let mut index: usize = 0; - while index < sql.len() { - let (name, end): (&str, usize) = match step(sql, index) { - ScanStep::Next(next) => { - index = next; - continue; - } - ScanStep::Unclosed(construct) => return Err(unclosed_message(context, construct)), - ScanStep::Intrinsic { name, end } => (name, end), - }; - let close: usize = match call_scan(sql, end) { - CallScan::Called(close) => close, - CallScan::NotCalled => { - return Err(format!("{context} intrinsic {name} must be called with ()")); - } - CallScan::Arguments => { - return Err(format!( - "{context} intrinsic {name} does not accept arguments" - )); - } - CallScan::Unclosed(construct) => { - return Err(unclosed_message( - &format!("{context} cursor intrinsic"), - construct, - )); - } - }; - replaced.push_str(&sql[last..index]); - let output_start: usize = replaced.len(); - replaced.push_str(&replacement(name)); - spans.push((index, close + 1, output_start, replaced.len())); - last = close + 1; - index = close + 1; - } - replaced.push_str(&sql[last..]); - Ok((replaced, spans)) -} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs new file mode 100644 index 0000000000..002783e8a1 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics_with_spans.rs @@ -0,0 +1,13 @@ +//! Cursor intrinsic replacement that reports where each call moved. + +use crate::cursor_intrinsics::_helpers::replacement::replaced_with_spans; +use crate::cursor_intrinsics::models::IntrinsicSpan; + +/// `sql` with each cursor intrinsic call replaced and each call's byte span, or the call's error. +pub fn replace_intrinsics_with_spans( + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, Vec), String> { + replaced_with_spans(sql, context, replacement) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs index 7474639f96..298b13f849 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs @@ -14,3 +14,12 @@ pub struct IntrinsicModel<'model> { pub incremental: bool, pub cursor: Option<&'model str>, } + +/// Where one replaced intrinsic call was in the source and is in the output, in bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct IntrinsicSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs index a21d7440bf..447cc98dbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/model_loop.rs @@ -17,7 +17,7 @@ use sqlbuild_analysis::compiled_project::models::{ use sqlbuild_analysis::sql_macros::main::moved_typed_calls::moved_typed_calls; use sqlbuild_analysis::sql_macros::main::spliced_typed_calls::spliced_typed_calls; use sqlbuild_analysis::sql_macros::models::{TextSplice, TypedCall}; -use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics_with_spans; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics_with_spans::replace_intrinsics_with_spans; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; @@ -983,9 +983,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::() } -/// Span rows as the spliced spans the scan-fault locator maps through. -/// The expression macro calls of the spliced SQL moved through the Python macro expansion and the -/// cursor intrinsic canonicalisation into the query's byte coordinates. +/// The spliced SQL's expression macro calls moved through macro expansion into the query's bytes. fn query_typed_calls( calls: Vec, (spliced, expanded, query): (&str, &str, &str), @@ -1012,16 +1010,15 @@ fn query_typed_calls( } let splices: Vec = spans .into_iter() - .map( - |(source_start, source_end, output_start, output_end)| TextSplice { - source: (source_start, source_end), - output: (output_start, output_end), - }, - ) + .map(|span| TextSplice { + source: (span.source_start, span.source_end), + output: (span.output_start, span.output_end), + }) .collect(); moved_typed_calls(calls, &splices) } +/// Span rows as the spliced spans the scan-fault locator maps through. fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() .map(|row| SplicedSpan { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs index b06802b89f..bbeb5bc735 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/sql_macro_host.rs @@ -96,8 +96,7 @@ pub(crate) fn is_sql_macro_reference(kind: u8) -> bool { kind == SQL_MACRO_REFERENCE_KIND } -/// The text replacing each SQL macro call starting at the given character offsets of `request.sql`, -/// with the expression macro calls each text holds. +/// The replacement text, with its expression macro calls, of each SQL macro call at `starts`. pub(crate) fn expanded_sql_macro_calls<'py>( request: &DeclarationExpansion<'_, 'py>, starts: &[usize], @@ -149,12 +148,7 @@ pub(crate) fn expanded_sql_macro_calls<'py>( Err(ExpansionFailure::Host(error)) => return Err(error), Err(ExpansionFailure::Macro(error)) => { let located: SqlMacroError = match host.call_location((call.start, call.end)) { - // A body that disagrees with its types is the macro file's error. - Some(location) - if [RETURNS_MISMATCH_CODE, BODY_TYPE_CODE].contains(&error.code) => - { - error.at(location) - } + Some(location) if is_macro_file_error(&error) => error.at(location), Some(location) => error.at_call(location), None => error, }; @@ -637,3 +631,8 @@ fn declaration_path( } Ok(None) } + +/// Whether `error` reports the macro's own body rather than one call of it. +fn is_macro_file_error(error: &SqlMacroError) -> bool { + [RETURNS_MISMATCH_CODE, BODY_TYPE_CODE].contains(&error.code) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py index 147ec3b2f3..06b81ef83a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py @@ -80,3 +80,26 @@ class SqlMacroCompileTestCase: expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = () inspected_paths: tuple[str, ...] = () + + +@dataclass(frozen=True) +class SqlMacroLineageTestCase: + """A column read through a SQL macro call, and the same column written inline.""" + + description: str + macro_files: dict[str, str] + inline_files: dict[str, str] + column: str + expected_edge: str + + +@dataclass(frozen=True) +class SqlMacroWarmCompileTestCase: + """A compiled project whose macro file is edited, then compiled warm, and what each prints.""" + + description: str + files: dict[str, str] + edited_path: str + replaced_text: tuple[str, str] + expected_cold_fragment: str + expected_warm_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index aca14a3337..a7fc17af82 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re import subprocess from collections.abc import Mapping from pathlib import Path @@ -232,12 +233,21 @@ def sql_text_macros(folder: str, names: list[str]) -> dict[str, str]: } -def text_macro( - name: str, returns: str | None, body: str, argument: str = "VARCHAR" -) -> dict[str, str]: - """The SQL macro `name` taking one `argument`-typed value and declaring `returns`, if any.""" +def text_macro(name: str, returns: str, body: str, argument: str = "VARCHAR") -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value and declaring `returns`.""" + + return _value_macro(name, f" returns {returns},\n", body, argument) + + +def untyped_text_macro(name: str, body: str, argument: str = "VARCHAR") -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value without declaring `returns`.""" + + return _value_macro(name, "", body, argument) + + +def _value_macro(name: str, declared: str, body: str, argument: str) -> dict[str, str]: + """The SQL macro `name` taking one `argument`-typed value, with its `declared` header lines.""" - declared: str = "" if returns is None else f" returns {returns},\n" return { f"macros/sql/{name}.sql": ( f'MACRO (\n description "Text from a value",\n arguments (value {argument}),\n' @@ -255,3 +265,23 @@ def contract_model(name: str, columns: str, sql: str) -> dict[str, str]: f" columns ({columns}),\n);\n\n{sql}" ) } + + +def typed_position_files(shape: str, sql: str, declared: str) -> dict[str, str]: + """The `orders` model over `sql`, contracted itself, contracted downstream, or uncontracted.""" + + downstream: dict[str, str] = contract_model( + "order_x", f"x (type {declared})", 'SELECT b.x FROM __ref("orders") AS b\n' + ) + shapes: dict[str, dict[str, str]] = { + "contracted": contract_model("orders", f"x (type {declared})", sql), + "downstream": {**orders_model(sql), **downstream}, + "uncontracted": orders_model(sql), + } + return shapes[shape] + + +def lineage_edges(printed: str) -> list[str]: + """The `source -> target` edges a list-format lineage command printed.""" + + return [edge.strip() for edge in re.findall(r"^.*->.*$", printed, flags=re.MULTILINE)] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index ca01b160e4..d51faac36c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -9,6 +9,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( SqlMacroCompileTestCase, + SqlMacroWarmCompileTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( contract_model, @@ -16,6 +17,7 @@ output, run_project, text_macro, + untyped_text_macro, with_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb @@ -26,6 +28,66 @@ ), } _UNTYPED_CALL: str = 'SELECT @sql.m(u.s) AS v FROM __ref("up") AS u\n' +_FIRST_OF: dict[str, str] = { + "macros/sql/first_of.sql": ( + 'MACRO (\n description "The first of two values",\n' + " arguments (value ANY, other ANY),\n returns TYPE OF value,\n);\n\n" + "list_extract([@value, @other], 1)\n" + ) +} +_FIRST_OF_UNTYPED: dict[str, str] = { + "macros/sql/first_of.sql": _FIRST_OF["macros/sql/first_of.sql"].replace( + " returns TYPE OF value,\n", "" + ) +} +_DAY: str = "CAST(o.{column} AS DATE)" +_DAYS: str = f"{_DAY.format(column='created_at')}, {_DAY.format(column='updated_at')}" +_TYPE_OF_POSITIONS: str = ( + "WITH c AS (\n" + f" SELECT o.order_id, @sql.first_of({_DAYS}) AS in_cte\n" + ' FROM __ref("stg_orders") AS o\n' + ")\n" + "SELECT\n" + " @sql.first_of(o.status, o.country_code) AS text_value,\n" + f" @sql.first_of({_DAYS}) AS day_value,\n" + " c.in_cte,\n" + f" COALESCE(@sql.first_of({_DAYS}), {_DAY.format(column='created_at')}) AS nested\n" + 'FROM __ref("stg_orders") AS o\nJOIN c ON c.order_id = o.order_id\n' +) +_TYPE_OF_COLUMNS: str = ( + "text_value (type VARCHAR), day_value (type DATE), in_cte (type DATE), nested (type DATE)" +) +_TYPE_OF_UNION: str = ( + 'SELECT CAST(NULL AS DATE) AS day_value FROM __ref("stg_orders") AS o\n' + "UNION ALL\n" + f'SELECT @sql.first_of({_DAYS}) AS day_value FROM __ref("stg_orders") AS o\n' +) +_VARIANT_PATHS: str = ( + "WITH raw AS (SELECT PARSE_JSON('{{\"a\": 1}}') AS payload)\n" + "SELECT @sql.payload_value({path}) AS v FROM raw\n" +) +_SNOWFLAKE_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" + ) +} +_ROUNDED: dict[str, str] = { + "macros/sql/rounded.sql": ( + 'MACRO (\n description "Rounded to places",\n' + " arguments (value ANY NUMERIC, places INT CONST DEFAULT 2),\n" + " returns TYPE OF value,\n);\n\nROUND(@value, @places)\n" + ), + "macros/sql/shouted.sql": ( + 'MACRO (\n description "Shouted",\n arguments (value ANY TEXT),\n' + " returns TYPE OF value,\n);\n\nUPPER(@value)\n" + ), + "macros/sql/month_of.sql": ( + 'MACRO (\n description "Month start",\n arguments (value ANY TEMPORAL),\n' + " returns TYPE OF value,\n);\n\nDATE_TRUNC('month', @value)\n" + ), +} @pytest.mark.parametrize( @@ -34,7 +96,7 @@ SqlMacroCompileTestCase( description="a_macro_without_returns_is_typed_from_its_body", files={ - **text_macro("m", None, "UPPER(@value)"), + **untyped_text_macro("m", "UPPER(@value)"), **contract_model( "orders", "v (type VARCHAR)", @@ -47,7 +109,7 @@ SqlMacroCompileTestCase( description="a_macro_without_returns_keeps_a_known_body_type_that_disagrees", files={ - **text_macro("m", None, "LENGTH(@value)"), + **untyped_text_macro("m", "LENGTH(@value)"), **contract_model( "orders", "v (type VARCHAR)", @@ -101,7 +163,7 @@ SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ - **text_macro("m", None, "@value", argument="ANY"), + **untyped_text_macro("m", "@value", argument="ANY"), **_UNTYPED_UPSTREAM, **contract_model("orders", "v (type BIGINT)", _UNTYPED_CALL), }, @@ -115,7 +177,7 @@ description="an_untyped_call_without_returns_suggests_returns_on_the_cast_rule", files={ **with_rules("SQBRCONTRACT105"), - **text_macro("m", None, "UPPER(@value)"), + **untyped_text_macro("m", "UPPER(@value)"), **contract_model( "orders", "v (type VARCHAR)", @@ -145,53 +207,6 @@ def test_given_optional_returns_when_compiling_then_calls_are_typed_and_checked( assert fragment not in output(result), output(result) -_FIRST_OF: dict[str, str] = { - "macros/sql/first_of.sql": ( - 'MACRO (\n description "The first of two values",\n' - " arguments (value ANY, other ANY),\n returns TYPE OF value,\n);\n\n" - "list_extract([@value, @other], 1)\n" - ) -} -_FIRST_OF_UNTYPED: dict[str, str] = { - "macros/sql/first_of.sql": _FIRST_OF["macros/sql/first_of.sql"].replace( - " returns TYPE OF value,\n", "" - ) -} -_DAY: str = "CAST(o.{column} AS DATE)" -_DAYS: str = f"{_DAY.format(column='created_at')}, {_DAY.format(column='updated_at')}" -_TYPE_OF_POSITIONS: str = ( - "WITH c AS (\n" - f" SELECT o.order_id, @sql.first_of({_DAYS}) AS in_cte\n" - ' FROM __ref("stg_orders") AS o\n' - ")\n" - "SELECT\n" - " @sql.first_of(o.status, o.country_code) AS text_value,\n" - f" @sql.first_of({_DAYS}) AS day_value,\n" - " c.in_cte,\n" - f" COALESCE(@sql.first_of({_DAYS}), {_DAY.format(column='created_at')}) AS nested\n" - 'FROM __ref("stg_orders") AS o\nJOIN c ON c.order_id = o.order_id\n' -) -_TYPE_OF_COLUMNS: str = ( - "text_value (type VARCHAR), day_value (type DATE), in_cte (type DATE), nested (type DATE)" -) -_TYPE_OF_UNION: str = ( - 'SELECT CAST(NULL AS DATE) AS day_value FROM __ref("stg_orders") AS o\n' - "UNION ALL\n" - f'SELECT @sql.first_of({_DAYS}) AS day_value FROM __ref("stg_orders") AS o\n' -) -_VARIANT_PATHS: str = ( - "WITH raw AS (SELECT PARSE_JSON('{{\"a\": 1}}') AS payload)\n" - "SELECT @sql.payload_value({path}) AS v FROM raw\n" -) -_SNOWFLAKE_PROJECT: dict[str, str] = { - "sqlbuild_project.toml": ( - 'name = "shop"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' - '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' - "[scopes]\nenforce_placement = false\n" - ) -} - - @pytest.mark.parametrize( "test_case", [ @@ -335,23 +350,6 @@ def test_given_returns_type_of_when_compiling_then_each_call_has_its_argument_ty assert fragment in output(result), output(result) -_ROUNDED: dict[str, str] = { - "macros/sql/rounded.sql": ( - 'MACRO (\n description "Rounded to places",\n' - " arguments (value ANY NUMERIC, places INT CONST DEFAULT 2),\n" - " returns TYPE OF value,\n);\n\nROUND(@value, @places)\n" - ), - "macros/sql/shouted.sql": ( - 'MACRO (\n description "Shouted",\n arguments (value ANY TEXT),\n' - " returns TYPE OF value,\n);\n\nUPPER(@value)\n" - ), - "macros/sql/month_of.sql": ( - 'MACRO (\n description "Month start",\n arguments (value ANY TEMPORAL),\n' - " returns TYPE OF value,\n);\n\nDATE_TRUNC('month', @value)\n" - ), -} - - @pytest.mark.parametrize( "test_case", [ @@ -445,22 +443,33 @@ def test_given_any_type_families_when_compiling_then_calls_and_bodies_are_checke assert fragment in output(result), output(result) +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroWarmCompileTestCase( + description="removing_returns_type_of", + files={ + **_FIRST_OF, + **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), + }, + edited_path="macros/sql/first_of.sql", + replaced_text=(" returns TYPE OF value,\n", ""), + expected_cold_fragment="0 errors, 0 warnings", + expected_warm_fragment="warning[K003]: column 'day_value'", + ), + ], + ids=lambda case: case.description, +) def test_given_a_changed_returns_form_when_compiling_warm_then_the_cache_is_not_reused( - tmp_path: Path, + test_case: SqlMacroWarmCompileTestCase, tmp_path: Path ) -> None: - project_dir, cold = run_project( - tmp_path=tmp_path, - files={ - **_FIRST_OF, - **contract_model("orders", "day_value (type DATE)", _TYPE_OF_UNION), - }, - command=("compile",), - ) - macro: Path = project_dir / "macros/sql/first_of.sql" - macro.write_text(_FIRST_OF_UNTYPED["macros/sql/first_of.sql"], encoding="utf-8") + project_dir, cold = run_project(tmp_path=tmp_path, files=test_case.files, command=("compile",)) + macro: Path = project_dir / test_case.edited_path + old, new = test_case.replaced_text + macro.write_text(macro.read_text(encoding="utf-8").replace(old, new), encoding="utf-8") warm: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "compile"), project_dir=project_dir ) - assert "0 errors, 0 warnings" in output(cold), output(cold) - assert "warning[K003]: column 'day_value'" in output(warm), output(warm) + assert test_case.expected_cold_fragment in output(cold), output(cold) + assert test_case.expected_warm_fragment in output(warm), output(warm) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index da75e0cdcd..97add8f60e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -3,170 +3,184 @@ from __future__ import annotations import subprocess -from dataclasses import dataclass from pathlib import Path import pytest +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, + SqlMacroLineageTestCase, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( BRACKET, contract_model, + inspected_text, + lineage_edges, orders_model, output, run_project, text_macro, + typed_position_files, + untyped_text_macro, with_rules, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb - -@dataclass(frozen=True) -class TypingPositionTestCase: - """One position of a call in a contracted model, and whether a known body type disagrees.""" - - description: str - sql: str - declared: str = "VARCHAR" - known_disagrees: bool = True - - _FROM: str = 'FROM __ref("stg_orders") AS o' +_COMPILED: str = "target/compiled/models/marts/orders.sql" +_CLEAN: tuple[str, ...] = ("0 errors, 0 warnings",) _MACROS: dict[str, str] = { **text_macro("status_bar", "VARCHAR", "bar(LENGTH(@value), 0, 10)"), **text_macro("same", "VARCHAR", "@value", argument="ANY"), **text_macro("shouted", "VARCHAR", "UPPER(@value)"), } -_POSITIONS: tuple[TypingPositionTestCase, ...] = ( - TypingPositionTestCase("main_whole_item", f"SELECT {{m}} AS x {_FROM}\n"), - TypingPositionTestCase( - "cte_item_read_by_name", f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT x FROM c\n" +_UNKNOWN_CALL: str = "@sql.status_bar(o.status)" +_KNOWN_CALL: str = "@sql.same(o.order_id)" +_DISAGREEING_POSITIONS: tuple[tuple[str, str, str, str], ...] = ( + ("main_whole_item", "contracted", f"SELECT {{m}} AS x {_FROM}\n", "VARCHAR"), + ( + "cte_item_read_by_name", + "contracted", + f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT x FROM c\n", + "VARCHAR", ), - TypingPositionTestCase( - "star_from_a_cte", f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT * FROM c\n" + ( + "star_from_a_cte", + "contracted", + f"WITH c AS (SELECT {{m}} AS x {_FROM})\nSELECT * FROM c\n", + "VARCHAR", ), - TypingPositionTestCase("derived_table", f"SELECT s.x FROM (SELECT {{m}} AS x {_FROM}) AS s\n"), - TypingPositionTestCase( - "scalar_subquery", f"SELECT (SELECT {{m}} {_FROM} LIMIT 1) AS x\n", known_disagrees=False + ( + "derived_table", + "contracted", + f"SELECT s.x FROM (SELECT {{m}} AS x {_FROM}) AS s\n", + "VARCHAR", ), - TypingPositionTestCase( + ( "nested_with", + "contracted", f"WITH c AS (WITH d AS (SELECT {{m}} AS x {_FROM}) SELECT x FROM d)\nSELECT x FROM c\n", + "VARCHAR", ), - TypingPositionTestCase( - "union_right_branch", - f"SELECT o.status AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", - known_disagrees=False, - ), - TypingPositionTestCase( - "union_left_branch", - f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT o.status AS x {_FROM}\n", - known_disagrees=False, - ), - TypingPositionTestCase( + ( "union_both_branches", + "contracted", f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + "VARCHAR", ), - TypingPositionTestCase( - "parenthesised_branch", - f"SELECT CAST(NULL AS VARCHAR) AS x {_FROM}\nUNION ALL\n(SELECT {{m}} AS x {_FROM})\n", - known_disagrees=False, - ), - TypingPositionTestCase( + ( "after_a_star_modifier", + "downstream", "WITH c AS (SELECT o.order_id, o.status " + _FROM + ")\n" "SELECT * EXCLUDE (order_id), {m} AS x FROM c AS o\n", + "VARCHAR", ), - TypingPositionTestCase( - "arithmetic", - f"SELECT LENGTH({{m}}) + 1 AS x {_FROM}\n", - declared="BIGINT", - known_disagrees=False, +) +_AGREEING_POSITIONS: tuple[tuple[str, str, str, str], ...] = ( + ( + "scalar_subquery", + "contracted", + f"SELECT (SELECT {{m}} {_FROM} LIMIT 1) AS x\n", + "VARCHAR", ), - TypingPositionTestCase( - "coalesce", f"SELECT COALESCE({{m}}, {{m}}) AS x {_FROM}\n", known_disagrees=False + ( + "union_right_branch", + "contracted", + f"SELECT o.status AS x {_FROM}\nUNION ALL\nSELECT {{m}} AS x {_FROM}\n", + "VARCHAR", ), - TypingPositionTestCase( - "case", - f"SELECT CASE WHEN o.order_id > 1 THEN {{m}} ELSE NULL END AS x {_FROM}\n", - known_disagrees=False, + ( + "union_left_branch", + "contracted", + f"SELECT {{m}} AS x {_FROM}\nUNION ALL\nSELECT o.status AS x {_FROM}\n", + "VARCHAR", ), - TypingPositionTestCase( - "function_argument", - f"SELECT LENGTH({{m}}) AS x {_FROM}\n", - declared="BIGINT", - known_disagrees=False, + ( + "parenthesised_branch", + "contracted", + f"SELECT CAST(NULL AS VARCHAR) AS x {_FROM}\nUNION ALL\n(SELECT {{m}} AS x {_FROM})\n", + "VARCHAR", ), - TypingPositionTestCase( - "aggregate_over_the_call", f"SELECT MAX({{m}}) AS x {_FROM}\n", known_disagrees=False + ("arithmetic", "contracted", f"SELECT LENGTH({{m}}) + 1 AS x {_FROM}\n", "BIGINT"), + ("coalesce", "contracted", f"SELECT COALESCE({{m}}, {{m}}) AS x {_FROM}\n", "VARCHAR"), + ( + "case", + "contracted", + f"SELECT CASE WHEN o.order_id > 1 THEN {{m}} ELSE NULL END AS x {_FROM}\n", + "VARCHAR", ), - TypingPositionTestCase( + ("function_argument", "contracted", f"SELECT LENGTH({{m}}) AS x {_FROM}\n", "BIGINT"), + ("aggregate_over_the_call", "contracted", f"SELECT MAX({{m}}) AS x {_FROM}\n", "VARCHAR"), + ( "window", + "contracted", f"SELECT MAX({{m}}) OVER (ORDER BY o.order_id) AS x {_FROM}\n", - known_disagrees=False, + "VARCHAR", ), - TypingPositionTestCase("unaliased_call", f"SELECT {{m}} {_FROM}\n", known_disagrees=False), + ("unaliased_call", "uncontracted", f"SELECT {{m}} {_FROM}\n", "VARCHAR"), ) -def _model(position: TypingPositionTestCase, call: str) -> dict[str, str]: - sql: str = position.sql.replace("{m}", call) - if position.description == "unaliased_call": - return orders_model(sql) - if position.description == "after_a_star_modifier": - return { - **orders_model(sql), - **contract_model( - "order_x", f"x (type {position.declared})", 'SELECT b.x FROM __ref("orders") AS b\n' - ), - } - return contract_model("orders", f"x (type {position.declared})", sql) - - -@pytest.mark.parametrize("position", _POSITIONS, ids=lambda case: case.description) +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description=description, + files={ + **_MACROS, + **typed_position_files(shape, sql.replace("{m}", _UNKNOWN_CALL), declared), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ) + for description, shape, sql, declared in _DISAGREEING_POSITIONS + _AGREEING_POSITIONS + ], + ids=lambda case: case.description, +) def test_given_an_unknown_body_when_compiling_then_the_call_takes_its_returns_type( - position: TypingPositionTestCase, tmp_path: Path + test_case: SqlMacroCompileTestCase, tmp_path: Path ) -> None: _, result = run_project( - tmp_path=tmp_path, - files={**_MACROS, **_model(position, "@sql.status_bar(o.status)")}, - command=("compile", "--no-cache"), + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") ) - assert result.returncode == 0, output(result) - assert "0 errors, 0 warnings" in output(result), output(result) + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) @pytest.mark.parametrize( - "position", - [position for position in _POSITIONS if position.known_disagrees], + "test_case", + [ + SqlMacroCompileTestCase( + description=description, + files={ + **_MACROS, + **typed_position_files(shape, sql.replace("{m}", _KNOWN_CALL), declared), + }, + expected_returncode=1, + expected_fragments=("error[K002]: column 'x' inferred as",), + ) + for description, shape, sql, declared in _DISAGREEING_POSITIONS + ], ids=lambda case: case.description, ) def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stands( - position: TypingPositionTestCase, tmp_path: Path + test_case: SqlMacroCompileTestCase, tmp_path: Path ) -> None: _, result = run_project( - tmp_path=tmp_path, - files={**_MACROS, **_model(position, "@sql.same(o.order_id)")}, - command=("compile", "--no-cache"), + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") ) - assert "error[K002]: column 'x' inferred as" in output(result), output(result) - - -@dataclass(frozen=True) -class TypingCaseTestCase: - """One project and what compiling it prints.""" - - description: str - files: dict[str, str] - expected_returncode: int - expected_fragments: tuple[str, ...] + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) @pytest.mark.parametrize( "test_case", [ - TypingCaseTestCase( + SqlMacroCompileTestCase( description="aggregate_inside_the_body", files={ **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), @@ -179,7 +193,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="aggregate_inside_the_body_still_needs_group_by", files={ **text_macro("summed", "BIGINT", "SUM(bar(LENGTH(@value), 0, 10))"), @@ -188,11 +202,11 @@ class TypingCaseTestCase: expected_returncode=1, expected_fragments=("error[B230]",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_nested_call_types_the_macro_calling_it", files={ **_MACROS, - **text_macro("outer_bar", None, "@sql.status_bar(@value)"), + **untyped_text_macro("outer_bar", "@sql.status_bar(@value)"), **contract_model( "orders", "x (type VARCHAR)", f"SELECT @sql.outer_bar(o.status) AS x {_FROM}\n" ), @@ -200,7 +214,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="an_any_argument_of_unknown_type_takes_returns", files={ **_MACROS, @@ -213,7 +227,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_known_identical_body", files={ **_MACROS, @@ -224,7 +238,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_call_inside_a_python_macro_argument_is_inferred_only", files={ **_MACROS, @@ -238,7 +252,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="an_outer_cast_wins_and_satisfies_the_cast_rule", files={ **with_rules("SQBRCONTRACT105"), @@ -252,7 +266,7 @@ class TypingCaseTestCase: expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_call_inside_arithmetic_needs_a_cast_for_the_cast_rule", files={ **with_rules("SQBRCONTRACT105"), @@ -269,7 +283,7 @@ class TypingCaseTestCase: "Wrap the complete expression in CAST(... AS BIGINT)", ), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_binding_error_after_a_call_is_located_where_it_was_written", files={ **_MACROS, @@ -278,13 +292,13 @@ class TypingCaseTestCase: expected_returncode=1, expected_fragments=("--> models/marts/orders.sql:3:42",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="the_reserved_analysis_function_is_rejected_in_a_model", files={**orders_model(f"SELECT __polyglot_typed(o.status, 'VARCHAR') AS x {_FROM}\n")}, expected_returncode=1, expected_fragments=("uses __polyglot_typed, a name reserved for SQL analysis",), ), - TypingCaseTestCase( + SqlMacroCompileTestCase( description="a_downstream_model_sees_the_returns_type", files={ **_MACROS, @@ -298,95 +312,108 @@ class TypingCaseTestCase: expected_returncode=1, expected_fragments=("error[K002]: column 'x' inferred as",), ), + SqlMacroCompileTestCase( + description="the_compiled_sql_has_no_marker", + files={ + **_MACROS, + **contract_model( + "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=("bar(LENGTH(o.status), 0, 10) AS x",), + unexpected_fragments=("__polyglot",), + inspected_paths=(_COMPILED,), + ), ], ids=lambda case: case.description, ) def test_given_a_typed_call_case_when_compiling_then_it_reports_as_expected( - test_case: TypingCaseTestCase, tmp_path: Path + test_case: SqlMacroCompileTestCase, tmp_path: Path ) -> None: - _, result = run_project( + project_dir, result = run_project( tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) assert result.returncode == test_case.expected_returncode, output(result) for fragment in test_case.expected_fragments: - assert fragment in output(result), output(result) + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed @pytest.mark.xfail( reason="a * REPLACE item takes its source column's type; a known polyglot gap", strict=True ) +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="a_call_in_a_star_replace", + files={ + **_MACROS, + **contract_model( + "orders", + "order_id (type VARCHAR)", + "WITH c AS (SELECT o.order_id " + _FROM + ")\n" + "SELECT * REPLACE (@sql.status_bar(CAST(order_id AS VARCHAR)) AS order_id) " + "FROM c\n", + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + ], + ids=lambda case: case.description, +) def test_given_a_call_in_a_star_replace_when_compiling_then_it_takes_its_returns_type( - tmp_path: Path, + test_case: SqlMacroCompileTestCase, tmp_path: Path ) -> None: _, result = run_project( - tmp_path=tmp_path, - files={ - **_MACROS, - **contract_model( - "orders", - "order_id (type VARCHAR)", - "WITH c AS (SELECT o.order_id " + _FROM + ")\n" - "SELECT * REPLACE (@sql.status_bar(CAST(order_id AS VARCHAR)) AS order_id) " - "FROM c\n", - ), - }, - command=("compile", "--no-cache"), + tmp_path=tmp_path, files=test_case.files, command=("compile", "--no-cache") ) - assert "0 errors, 0 warnings" in output(result), output(result) + assert result.returncode == test_case.expected_returncode, output(result) + for fragment in test_case.expected_fragments: + assert fragment in output(result), output(result) -def test_given_typed_calls_when_compiling_then_the_compiled_sql_has_no_marker( - tmp_path: Path, +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroLineageTestCase( + description="an_identity_macro", + macro_files={ + **text_macro("ident", "BIGINT", "@value", argument="BIGINT"), + **orders_model(f"SELECT @sql.ident(o.order_id) AS x {_FROM}\n"), + }, + inline_files=orders_model(f"SELECT o.order_id AS x {_FROM}\n"), + column="orders.x", + expected_edge="stg_orders.order_id -> orders.x", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_call_when_reading_lineage_then_it_matches_the_inline_sql( + test_case: SqlMacroLineageTestCase, tmp_path: Path ) -> None: - project_dir, result = run_project( - tmp_path=tmp_path, - files={ - **_MACROS, - **contract_model( - "orders", "x (type VARCHAR)", f"SELECT @sql.status_bar(o.status) AS x {_FROM}\n" - ), - }, - command=("compile", "--no-cache"), - ) - compiled: list[Path] = [ - path for path in (project_dir / "target").rglob("orders.sql") if "compiled" in str(path) - ] - - assert result.returncode == 0, output(result) - assert compiled, output(result) - text: str = compiled[0].read_text(encoding="utf-8") - assert "bar(LENGTH(o.status), 0, 10) AS x" in text, text - assert "__polyglot" not in text, text - - -def test_given_a_call_when_reading_lineage_then_it_matches_the_inline_sql(tmp_path: Path) -> None: macro_dir, _ = run_project( - tmp_path=tmp_path / "macro", - files={ - **text_macro("ident", "BIGINT", "@value", argument="BIGINT"), - **orders_model(f"SELECT @sql.ident(o.order_id) AS x {_FROM}\n"), - }, - command=("compile", "--no-cache"), + tmp_path=tmp_path / "macro", files=test_case.macro_files, command=("compile", "--no-cache") ) inline_dir, _ = run_project( tmp_path=tmp_path / "inline", - files=orders_model(f"SELECT o.order_id AS x {_FROM}\n"), + files=test_case.inline_files, command=("compile", "--no-cache"), ) - lineage = ("--no-color", "lineage", "orders.x", "--format", "list") + lineage: tuple[str, ...] = ("--no-color", "lineage", test_case.column, "--format", "list") through_macro: subprocess.CompletedProcess[str] = run_sqb( command=lineage, project_dir=macro_dir ) inline: subprocess.CompletedProcess[str] = run_sqb(command=lineage, project_dir=inline_dir) assert through_macro.returncode == 0, output(through_macro) - assert "stg_orders.order_id -> orders.x" in through_macro.stdout, output(through_macro) - assert _edges(through_macro.stdout) == _edges(inline.stdout) - - -def _edges(printed: str) -> list[str]: - """The lineage edges a list-format lineage command printed.""" - - return [line.strip() for line in printed.splitlines() if "->" in line] + assert test_case.expected_edge in through_macro.stdout, output(through_macro) + assert lineage_edges(through_macro.stdout) == lineage_edges(inline.stdout), output(inline) From 5072a63def871fe58c1f24d7760bff823cac94de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:31:43 +0100 Subject: [PATCH 13/28] fix: keep normalizing decimal types wider than the dialect holds --- .../src/type_system/_helpers/parsed.rs | 22 +++++++++++++---- ...rized_nested_padded_and_invalid_types.json | 24 +++++++++---------- .../test_native_type_normalization.py | 9 +++++++ 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs index e37fddd4b9..1b4007841f 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs @@ -1,6 +1,8 @@ //! Python's `_normalize_with_polyglot`: the parsed type's SQL and arguments. -use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions}; +use polyglot_sql::{ + ComplexityGuardOptions, DataType, Dialect, DialectType, Expression, ParseOptions, +}; use serde_json::{Map, Value}; use crate::type_system::_helpers::fallback::{ @@ -34,6 +36,20 @@ pub(crate) fn parse_type(type_sql: &str, dialect: &Dialect) -> Result Result { + match polyglot.generate(expression) { + Ok(generated) => Ok(generated), + Err(polyglot_sql::error::Error::Unsupported { .. }) => Dialect::get(DialectType::Generic) + .generate(expression) + .map_err(|error| TypeNormalizationError::Generation(error.to_string())), + Err(error) => Err(TypeNormalizationError::Generation(error.to_string())), + } +} + /// Python's `_normalized_from_parsed_type`. pub(crate) fn normalize_parsed( parsed: DataType, @@ -42,9 +58,7 @@ pub(crate) fn normalize_parsed( ) -> Result { let args: Map = shallow_args(&parsed); let expression: Expression = Expression::DataType(parsed); - let generated: String = polyglot - .generate(&expression) - .map_err(|error| TypeNormalizationError::Generation(error.to_string()))?; + let generated: String = generated_type(&expression, polyglot)?; let normalized_name: String = upper(&generated).replace(' ', ""); let dtype_name: String = polyglot_type_name(&args); let params: Vec = polyglot_type_params(&args); diff --git a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json index a7ccced078..30eded6455 100644 --- a/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json +++ b/tests/goldens/python_oracle/types_generated_seeded_scalar_parameterized_nested_padded_and_invalid_types.json @@ -12497,16 +12497,16 @@ [ "type", { -"DuckDB": "9c4e957c3856f1159283313a", +"DuckDB": "8731ee5ed03816bf6b4a8464", "None": "8731ee5ed03816bf6b4a8464", "bigquery": "657a69dabb6a9b2c7b834d41", "databricks": "606d813c3f1dfcf4921045c8", -"duckdb": "9c4e957c3856f1159283313a", +"duckdb": "8731ee5ed03816bf6b4a8464", "generic": "8731ee5ed03816bf6b4a8464", "motherduck": "901c5133ed47d4769286e860", "mysql": "b9f1a78bcda3bc968346cc10", -"postgres": "9c4e957c3856f1159283313a", -"postgresql": "9c4e957c3856f1159283313a", +"postgres": "8731ee5ed03816bf6b4a8464", +"postgresql": "8731ee5ed03816bf6b4a8464", "snowflake": "cd2eb5ae8b54388fde5d5737", "tsql": "8619573249fac6f54d0bc771" } @@ -12718,16 +12718,16 @@ [ "type", { -"DuckDB": "0da0fad3107f7a26744c9411", +"DuckDB": "4172129e8d84a0da597bc612", "None": "4172129e8d84a0da597bc612", "bigquery": "d138b3b3355094a33a14355a", "databricks": "4172129e8d84a0da597bc612", -"duckdb": "0da0fad3107f7a26744c9411", +"duckdb": "4172129e8d84a0da597bc612", "generic": "4172129e8d84a0da597bc612", "motherduck": "901c5133ed47d4769286e860", "mysql": "6c03a3450ea4742ee7396e3c", -"postgres": "0da0fad3107f7a26744c9411", -"postgresql": "0da0fad3107f7a26744c9411", +"postgres": "4172129e8d84a0da597bc612", +"postgresql": "4172129e8d84a0da597bc612", "snowflake": "e21aafb74f6e5c5e181abd5e", "tsql": "2e261f2f5a01e0e0f4a34a7a" } @@ -16458,16 +16458,16 @@ [ "type", { -"DuckDB": "e1554f02dea662c971e7b9b9", +"DuckDB": "6f851103102e7b986535376f", "None": "6f851103102e7b986535376f", "bigquery": "58b7e0908df666938398205f", "databricks": "6f851103102e7b986535376f", -"duckdb": "e1554f02dea662c971e7b9b9", +"duckdb": "6f851103102e7b986535376f", "generic": "6f851103102e7b986535376f", "motherduck": "901c5133ed47d4769286e860", "mysql": "1a853242fb56557e2865a86a", -"postgres": "a0b34c5b4e5ace89b22c6971", -"postgresql": "a0b34c5b4e5ace89b22c6971", +"postgres": "6f851103102e7b986535376f", +"postgresql": "6f851103102e7b986535376f", "snowflake": "87800133cac38d9119180c0a", "tsql": "0e9ef5373e5d52970e8e1656" } diff --git a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py index ee1cdf607d..79cb00a903 100644 --- a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py +++ b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_normalization.py @@ -100,6 +100,15 @@ def test_given_generated_types_when_normalizing_then_every_known_dialect_answers ), expected_parse_error_logged=False, ), + TypeNormalizationTestCase( + description="a decimal wider than the dialect holds keeps its precision", + type_sql="DECIMAL(39, 2)", + dialect="duckdb", + expected_type=NormalizedType( + normalized_name="DECIMAL(39,2)", family=TypeFamily.DECIMAL, precision=39, scale=2 + ), + expected_parse_error_logged=False, + ), TypeNormalizationTestCase( description="spaced decimal parameters", type_sql=" decimal( 10 , 2 ) ", From a532c130453b422881b7c5240e7f58c5bd63431b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:31:44 +0100 Subject: [PATCH 14/28] test: keep the invalid macro inventory case invalid now that returns is optional --- .../unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 78b5d15fe4..8339b61cba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -777,7 +777,7 @@ def test_given_sql_with_at_tokens_when_finding_macro_calls_then_returns_real_uni 'MACRO (description "Valid", arguments (v BIGINT), returns BIGINT);\n\n@v + 1\n' ), "macros/sql/z_invalid.sql": ( - 'MACRO (description "Invalid", arguments (v BIGINT));\n\n@v + 1\n' + 'MACRO (description "Invalid", returns BIGINT);\n\nWITH a AS (SELECT 1)\n' ), }, expected_exports=("a_valid",), From 9b9e9cba31fe0ed556224e30f0676d91dd7ab8b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 08:47:09 +0100 Subject: [PATCH 15/28] test: record compiler goldens for the SQL macro returns and family checks --- .../_helpers/corpus/render_failure_cases.py | 6 ++++-- .../sql-macro-body-misuses-an-any-family.json | 19 +++++++++++++++++++ .../failure/sql-macro-cte-name-clash.json | 2 +- .../failure/sql-macro-invalid-definition.json | 4 ++-- ...sql-macro-returns-disagrees-with-body.json | 19 +++++++++++++++++++ 5 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json create mode 100644 tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index 9f5149616b..7aa0034670 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -683,10 +683,12 @@ def _sql_macro_cases() -> tuple[FailureCase, ...]: failure_case( name="sql-macro-invalid-definition", expected_code="P015", - expected_message="expression macro x() needs returns in its MACRO header", + expected_message="returns TYPE OF w names no argument of macro x()", files=_sql_macro_columns( columns="@sql.x(amount) AS amount, status", - files=_x_macro(arguments="v DOUBLE", returns="", body="@v + 1\n"), + files=_x_macro( + arguments="v DOUBLE", returns=", returns TYPE OF w", body="@v + 1\n" + ), ), ), failure_case( diff --git a/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json b/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json new file mode 100644 index 0000000000..716f276699 --- /dev/null +++ b/tests/goldens/compiler/failure/sql-macro-body-misuses-an-any-family.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P025", + "message": "body of macro x() does not accept every value of v ANY NUMERIC: Function 'upper' argument 1 expects String, found numeric\n model: stg_orders\n --> models/staging/_sqlbuild/_macros/sql/x.sql:3:1\n --> models/staging/stg_orders.sql:5:31", + "help": "an ANY family argument may be any type of its family; the body is checked with the family's widest type, so CAST the argument or narrow its declared type" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json b/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json index 569a956a00..360c1a6978 100644 --- a/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json +++ b/tests/goldens/compiler/failure/sql-macro-cte-name-clash.json @@ -9,7 +9,7 @@ "severity": "error", "code": "P018", "message": "CTE \"live\" from macro live_orders() clashes with a CTE already defined in model:stg_orders\n model: stg_orders\n --> models/staging/stg_orders.sql:7:1\n --> models/staging/_sqlbuild/_macros/sql/live_orders.sql:6:6", - "help": "rename one of the CTEs; a model can't call a CTE-list macro whose CTE names it already uses" + "help": "pass __cte := to give the CTEs one call inserts their own names, as in @sql.live_orders(..., __cte := first_orders), or rename one of the CTEs" } ] } diff --git a/tests/goldens/compiler/failure/sql-macro-invalid-definition.json b/tests/goldens/compiler/failure/sql-macro-invalid-definition.json index bce56b86fc..a85b291acf 100644 --- a/tests/goldens/compiler/failure/sql-macro-invalid-definition.json +++ b/tests/goldens/compiler/failure/sql-macro-invalid-definition.json @@ -8,8 +8,8 @@ "phase": "compile", "severity": "error", "code": "P015", - "message": "expression macro x() needs returns in its MACRO header\n --> models/staging/_sqlbuild/_macros/sql/x.sql:1:1", - "help": "declare the type the expression returns, such as returns DECIMAL(18, 2)" + "message": "returns TYPE OF w names no argument of macro x()\n --> models/staging/_sqlbuild/_macros/sql/x.sql:1:63", + "help": "name one of its arguments: v" } ] } diff --git a/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json b/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json new file mode 100644 index 0000000000..e9042051be --- /dev/null +++ b/tests/goldens/compiler/failure/sql-macro-returns-disagrees-with-body.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P024", + "message": "macro x() declares returns DOUBLE, but its body is VARCHAR\n model: stg_orders\n --> models/staging/_sqlbuild/_macros/sql/x.sql:3:1\n --> models/staging/stg_orders.sql:5:31", + "help": "declare returns VARCHAR, CAST the body to DOUBLE, or remove returns to use the inferred type" + } + ] + } + ], + "compiled": {}, + "manifest": null +} From 74fe17bc440c459d59de4350dd1053850c769faa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:12:33 +0100 Subject: [PATCH 16/28] fix: skip the returns check when a body calls a function of unknown type --- .../main/annotated_output_types.rs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs index 1d22c5d16d..5ba4a6b6ff 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs @@ -1,10 +1,12 @@ use std::collections::HashMap; +use polyglot_sql::expressions::DataType; +use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_outputs}; -/// The output columns and types of the one query in `sql`, typed by full type annotation. +/// The output types of the one query in `sql` by full annotation; `None` if a call is untyped. #[must_use] pub fn annotated_output_types( sql: &str, @@ -32,9 +34,27 @@ pub fn annotated_output_types( Some(&mapping as &dyn polyglot_sql::schema::Schema), Some(dialect), ); + if has_untyped_call(&expression) { + return None; + } let mut outputs: Vec<(String, Option)> = Vec::new(); for (name, data_type) in infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) { outputs.push((name, data_type.filter(|value| value != NULL_TYPE))); } Some(outputs) } + +/// Whether a function call under `node`, outside a typed marker, has no known result type. +fn has_untyped_call(node: &Expression) -> bool { + match node { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, + Expression::Function(_) + | Expression::AggregateFunction(_) + | Expression::WindowFunction(_) + if node.inferred_type().is_none_or(DataType::contains_unknown) => + { + true + } + _ => node.children().into_iter().any(has_untyped_call), + } +} From 32cd1f1d1490abae41f4b97bfb36f094a05e4d06 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 09:12:34 +0100 Subject: [PATCH 17/28] fix: judge a whole typed SQL macro call by its returns, not its body's cast --- .../rules/_helpers/explicit_output_types.rs | 4 +--- .../sql_macros/test_sql_macro_returns.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs index f217c67301..74c57fecbc 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/explicit_output_types.rs @@ -187,9 +187,7 @@ impl OutputRule<'_> { .cast_type .as_ref() .is_some_and(|data_type| type_spelling_equal(data_type, &column.data_type)); - let call: Option<&TypedCall> = (!explicit_cast) - .then(|| self.whole_call(expression)) - .flatten(); + let call: Option<&TypedCall> = self.whole_call(expression); let passes = cast_matches_contract || column.type_proven && direct && !scope.set_branch || column.type_proven diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index d51faac36c..b3780eecaa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -160,6 +160,16 @@ expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), + SqlMacroCompileTestCase( + description="a_body_with_an_untyped_function_is_not_checked_against_returns", + files={ + **_SNOWFLAKE_PROJECT, + **text_macro("m", "FLOAT", "COALESCE(TRY_TO_DOUBLE(@value), 0)"), + **orders_model('SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ @@ -190,6 +200,20 @@ "Declare returns VARCHAR in SQL macro m(), or wrap the call in CAST(... AS VARCHAR).", ), ), + SqlMacroCompileTestCase( + description="a_cast_inside_a_typed_body_is_not_the_outputs_cast", + files={ + **with_rules("SQBRCONTRACT105"), + **text_macro("m", "VARCHAR", "CAST(LENGTH(@value) AS TEXT)"), + **contract_model( + "orders", + "v (type VARCHAR)", + 'SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), ], ids=lambda case: case.description, ) From 20b4d179b1bc043d49eeb6479c44860614f49c2c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:47:38 +0100 Subject: [PATCH 18/28] build: point polyglot at the typed markers with transparent grouping and shape-aware refinement --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6263de95b1..98be220e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ee40a7246867f6da80fcf38b932c1eb5886a415e#ee40a7246867f6da80fcf38b932c1eb5886a415e" +source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index c68a46aea3..cf4ffbceb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "ee40a7246867f6da80fcf38b932c1eb5886a415e", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "ee40a7246867f6da80fcf38b932c1eb5886a415e", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } From 376a05ed1581f105d45cfe23bd0c3324a457772b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:00:43 +0100 Subject: [PATCH 19/28] fix: skip the returns check where a body's type is only a guess --- .../main/annotated_output_types.rs | 27 ++++++++++----- .../main/compile/sql_macros/helpers.py | 12 +++++++ .../sql_macros/test_sql_macro_returns.py | 33 +++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs index 5ba4a6b6ff..eabbb7c0e9 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use polyglot_sql::expressions::DataType; +use polyglot_sql::optimizer::annotate_types::function_result_is_known; use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; @@ -34,7 +35,7 @@ pub fn annotated_output_types( Some(&mapping as &dyn polyglot_sql::schema::Schema), Some(dialect), ); - if has_untyped_call(&expression) { + if has_untyped_call(&expression, dialect) { return None; } let mut outputs: Vec<(String, Option)> = Vec::new(); @@ -44,17 +45,27 @@ pub fn annotated_output_types( Some(outputs) } -/// Whether a function call under `node`, outside a typed marker, has no known result type. -fn has_untyped_call(node: &Expression) -> bool { +/// Whether a call under `node`, outside a typed marker, has an unknown or only guessed type. +fn has_untyped_call(node: &Expression, dialect: DialectType) -> bool { match node { Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, - Expression::Function(_) - | Expression::AggregateFunction(_) - | Expression::WindowFunction(_) - if node.inferred_type().is_none_or(DataType::contains_unknown) => + Expression::Function(function) + if !function_result_is_known(function, dialect) || is_unknown(node) => { true } - _ => node.children().into_iter().any(has_untyped_call), + Expression::AggregateFunction(_) | Expression::WindowFunction(_) if is_unknown(node) => { + true + } + _ => node + .children() + .into_iter() + .any(|child| has_untyped_call(child, dialect)), } } + +/// Whether annotation left `node` without a type, or with a wholly unknown one. +fn is_unknown(node: &Expression) -> bool { + node.inferred_type() + .is_none_or(|data_type| *data_type == DataType::Unknown) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index a7fc17af82..60b69fa90d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -285,3 +285,15 @@ def lineage_edges(printed: str) -> list[str]: """The `source -> target` edges a list-format lineage command printed.""" return [edge.strip() for edge in re.findall(r"^.*->.*$", printed, flags=re.MULTILINE)] + + +def adapter_project(adapter: str) -> dict[str, str]: + """The orders project configured for `adapter`, compiled without a warehouse.""" + + return { + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{adapter}"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "warehouse"\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" + ) + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index b3780eecaa..90db424af9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -12,6 +12,7 @@ SqlMacroWarmCompileTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + adapter_project, contract_model, orders_model, output, @@ -170,6 +171,38 @@ expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), + SqlMacroCompileTestCase( + description="a_guessed_date_trunc_type_on_databricks_is_not_checked_against_returns", + files={ + **adapter_project("databricks"), + **text_macro("m", "TIMESTAMP", "DATE_TRUNC('DAY', @value)", argument="TIMESTAMP"), + **orders_model('SELECT @sql.m(o.created_at) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_guessed_regexp_contains_type_on_bigquery_is_not_checked_against_returns", + files={ + **adapter_project("bigquery"), + **text_macro("m", "BOOL", "REGEXP_CONTAINS(@value, 'a')", argument="VARCHAR"), + **orders_model('SELECT @sql.m(o.status) AS v FROM __ref("stg_orders") AS o\n'), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), + SqlMacroCompileTestCase( + description="a_user_defined_function_on_bigquery_is_not_checked_against_returns", + files={ + **adapter_project("bigquery"), + **text_macro("m", "STRING", "my_udf(@value)", argument="BIGINT"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ From aff024dc928085835e0142b0bd6e6ffbdfdef67c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:01:56 +0100 Subject: [PATCH 20/28] test: cover grouping keys written through typed SQL macro calls --- .../test_sql_macro_typing_matrix.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index 97add8f60e..cad947b5b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -312,6 +312,72 @@ def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stan expected_returncode=1, expected_fragments=("error[K002]: column 'x' inferred as",), ), + SqlMacroCompileTestCase( + description="a_grouping_key_written_through_a_call", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT o.amount_cents + 1 AS k, COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_in_select_and_grouping_key", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT @sql.plus_one(o.amount_cents) AS k, COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_selected_and_grouped_written_out", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT @sql.plus_one(o.amount_cents) AS k, COUNT(*) AS n {_FROM} GROUP BY o.amount_cents + 1\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_call_in_a_rollup", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT o.amount_cents + 1 AS k, COUNT(*) AS n {_FROM} GROUP BY ROLLUP (@sql.plus_one(o.amount_cents))\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_having_over_a_called_grouping_key", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT COUNT(*) AS n {_FROM} GROUP BY @sql.plus_one(o.amount_cents) HAVING o.amount_cents + 1 > 0\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), + SqlMacroCompileTestCase( + description="a_distinct_ordered_by_a_call", + files={ + **text_macro("plus_one", "BIGINT", "@value + 1", argument="BIGINT"), + **orders_model( + f"SELECT DISTINCT o.amount_cents + 1 AS k {_FROM} ORDER BY @sql.plus_one(o.amount_cents)\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + ), SqlMacroCompileTestCase( description="the_compiled_sql_has_no_marker", files={ From d9e4f74860abd0bac0cca8eecec3497036e6ab53 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:13:50 +0100 Subject: [PATCH 21/28] fix: rename __cte references by the CTE each name binds to in its scope --- .../_helpers/rendering/cte_rename.rs | 156 +++++++++++++++--- .../main/compile/sql_macros/helpers.py | 15 ++ .../sql_macros/test_sql_macro_cte_names.py | 57 +++++++ 3 files changed, 201 insertions(+), 27 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs index 29118b660e..daf11aec09 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs @@ -117,40 +117,20 @@ pub(crate) fn renamed_ctes(body: &ExpandedText, name: &str) -> Result = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); - let mut aliases: Vec = Vec::new(); - for node in statement.dfs() { - if let Expression::Table(table) = node - && let Some(alias) = &table.alias - { - aliases.push(alias.name.clone()); - } - } let mut identifiers: Vec<&Identifier> = with.ctes.iter().map(|cte| &cte.alias).collect(); - for node in statement.dfs() { - match node { - Expression::Table(table) if table.schema.is_none() && table.catalog.is_none() => { - identifiers.push(&table.name); - } - Expression::Column(column) => identifiers.extend(column.table.as_ref()), - Expression::Star(star) => identifiers.extend(star.table.as_ref()), - _ => {} - } - } - let shadowed = |identifier: &Identifier| { - aliases - .iter() - .any(|alias| alias.eq_ignore_ascii_case(&identifier.name)) + let mut walk = CteReferences { + frames: Vec::new(), + scopes: Vec::new(), + found: Vec::new(), }; + walk.visit(&statement, true); + identifiers.extend(walk.found); let offsets: CharOffsets = CharOffsets::new(&parsed_text); let mut edits: Vec<(usize, usize, String)> = Vec::new(); - for (position, identifier) in identifiers.into_iter().enumerate() { - let is_definition: bool = position < with.ctes.len(); + for identifier in identifiers { let Some(new_name) = renamed(&ctes, name, &identifier.name) else { continue; }; - if !is_definition && shadowed(identifier) { - continue; - } let Some(span) = identifier.span else { continue; }; @@ -188,6 +168,128 @@ pub(crate) fn renamed_ctes(body: &ExpandedText, name: &str) -> Result { + /// The CTE names of each enclosing `WITH`, innermost last, and whether they are the body's. + frames: Vec<(Vec, bool)>, + /// The relations of each enclosing query, innermost last. + scopes: Vec>, + found: Vec<&'a Identifier>, +} + +impl<'a> CteReferences<'a> { + fn visit(&mut self, node: &'a Expression, top: bool) { + let with = match node { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(union) => union.with.as_ref(), + Expression::Intersect(intersect) => intersect.with.as_ref(), + Expression::Except(except) => except.with.as_ref(), + _ => None, + }; + if let Some(with) = with { + let names: Vec = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); + self.frames.push((names, top)); + } + let scoped: bool = if let Expression::Select(select) = node { + let sources: Vec = select + .from + .iter() + .flat_map(|from| &from.expressions) + .chain(select.joins.iter().map(|join| &join.this)) + .filter_map(|source| self.scope_source(source)) + .collect(); + self.scopes.push(sources); + true + } else { + false + }; + match node { + Expression::Table(table) if self.is_body_cte(table) => self.found.push(&table.name), + Expression::Column(column) => self.qualifier(column.table.as_ref()), + Expression::Star(star) => self.qualifier(star.table.as_ref()), + _ => {} + } + for child in node.children() { + self.visit(child, false); + } + if scoped { + self.scopes.pop(); + } + if with.is_some() { + self.frames.pop(); + } + } + + /// Whether `table` is an unqualified name whose innermost binding is one of the body's CTEs. + fn is_body_cte(&self, table: &polyglot_sql::expressions::TableRef) -> bool { + if table.schema.is_some() || table.catalog.is_some() { + return false; + } + self.frames + .iter() + .rev() + .find(|(names, _)| includes_name(names, &table.name.name)) + .is_some_and(|(_, body)| *body) + } + + /// The name a FROM or JOIN item is read as, and whether it is an unaliased body CTE. + fn scope_source(&self, source: &Expression) -> Option { + match source { + Expression::Table(table) => Some(match &table.alias { + Some(alias) => ScopeSource { + name: alias.name.clone(), + body_cte: false, + }, + None => ScopeSource { + name: table.name.name.clone(), + body_cte: self.is_body_cte(table), + }, + }), + Expression::Subquery(subquery) => subquery.alias.as_ref().map(|alias| ScopeSource { + name: alias.name.clone(), + body_cte: false, + }), + Expression::Paren(paren) => self.scope_source(&paren.this), + _ => None, + } + } + + /// Records `qualifier` when the relation it names, innermost scope first, is a body CTE. + fn qualifier(&mut self, qualifier: Option<&'a Identifier>) { + let Some(qualifier) = qualifier else { + return; + }; + let source: Option<&ScopeSource> = self + .scopes + .iter() + .rev() + .find_map(|sources| source_named(sources, &qualifier.name)); + if source.is_some_and(|source| source.body_cte) { + self.found.push(qualifier); + } + } +} + +/// Whether `names` holds `name`, ignoring case. +fn includes_name(names: &[String], name: &str) -> bool { + names + .iter() + .any(|defined| defined.eq_ignore_ascii_case(name)) +} + +/// The relation of `sources` read as `name`, ignoring case. +fn source_named<'s>(sources: &'s [ScopeSource], name: &str) -> Option<&'s ScopeSource> { + sources + .iter() + .find(|source| source.name.eq_ignore_ascii_case(name)) +} + /// The one statement `text` parses to, in the first dialect that parses it. fn parsed(text: &str) -> Result { let mut first_error: Option = None; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index 60b69fa90d..b071b39384 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -297,3 +297,18 @@ def adapter_project(adapter: str) -> dict[str, str]: "[scopes]\nenforce_placement = false\n" ) } + + +def picked_orders(body: str) -> dict[str, str]: + """The CTE-list macro `pick` over relation `src` with `body`, called once as `__cte := a`.""" + + return { + "macros/sql/pick.sql": ( + 'MACRO (\n description "Picked orders",\n' + " arguments (src RELATION (order_id BIGINT)),\n);\n\n" + body + "\n" + ), + **orders_model( + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.pick(raw, __cte := a)\nSELECT * FROM a\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py index ce3b2dc545..e00d6a2824 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py @@ -19,6 +19,7 @@ inspected_text, orders_model, output, + picked_orders, run_project, text_macro, ) @@ -66,6 +67,62 @@ query="SELECT live_rows FROM orders", expected_rows=[(3,)], ), + SqlMacroBuildTestCase( + description="a_cte_read_under_its_own_name_as_alias", + files=picked_orders( + "WITH live AS (SELECT * FROM @src),\nlatest AS (SELECT live.order_id FROM live AS live)" + ), + query="SELECT COUNT(*) FROM orders", + expected_rows=[(4,)], + ), + SqlMacroBuildTestCase( + description="an_alias_named_like_a_cte_elsewhere", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 1),\nlatest AS (SELECT live.order_id FROM live JOIN @src AS other ON live.order_id = other.order_id),\nfinal AS (SELECT o.order_id FROM latest AS o JOIN @src AS live ON live.order_id = o.order_id)" + ), + query="SELECT COUNT(*) FROM orders", + expected_rows=[(8,)], + ), + SqlMacroBuildTestCase( + description="a_nested_with_shadowing_a_cte", + files=picked_orders( + "WITH live AS (SELECT * FROM @src),\nlatest AS (WITH live AS (SELECT 42 AS order_id) SELECT order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(42,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_derived_table", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT s.order_id FROM (SELECT live.order_id FROM live) AS s)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_set_operation", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT order_id FROM live UNION ALL SELECT live.order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,), (2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM @src AS r JOIN live ON live.order_id = r.order_id)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_correlated_subquery", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT r.order_id FROM @src AS r WHERE EXISTS (SELECT 1 FROM live WHERE live.order_id = r.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), ], ids=lambda case: case.description, ) From a6c38b468457cb0f94f831e6d23c1502b3beedb6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:27:48 +0100 Subject: [PATCH 22/28] fix: check returns against the shape of a partly unknown macro body --- .../_helpers/annotated_types.rs | 47 ++++++++ .../main/annotated_output_types.rs | 48 +++----- .../src/query_analysis/mod.rs | 3 + .../src/query_analysis/types.rs | 6 + .../_helpers/typing/returns_check.rs | 104 +++++++++++++----- .../src/sql_macros/models.rs | 14 +++ .../sql_macros/test_sql_macro_returns.py | 26 +++++ 7 files changed, 189 insertions(+), 59 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs create mode 100644 crates/sqlbuild-analysis/src/query_analysis/types.rs diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs new file mode 100644 index 0000000000..28a8d6b503 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/annotated_types.rs @@ -0,0 +1,47 @@ +//! Facts of a fully annotated query: which calls annotation could not type, and output types. + +use polyglot_sql::expressions::DataType; +use polyglot_sql::optimizer::annotate_types::function_result_is_known; +use polyglot_sql::traversal::ExpressionWalk; +use polyglot_sql::{DialectType, Expression}; + +/// Whether a call under `node`, outside a typed marker, has an unknown or only guessed type. +pub(crate) fn has_untyped_call(node: &Expression, dialect: DialectType) -> bool { + match node { + Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, + Expression::Function(function) + if !function_result_is_known(function, dialect) || is_unknown(node) => + { + true + } + Expression::AggregateFunction(_) | Expression::WindowFunction(_) if is_unknown(node) => { + true + } + _ => node + .children() + .into_iter() + .any(|child| has_untyped_call(child, dialect)), + } +} + +/// Whether annotation left `node` without a type, or with a wholly unknown one. +fn is_unknown(node: &Expression) -> bool { + node.inferred_type() + .is_none_or(|data_type| *data_type == DataType::Unknown) +} + +/// The annotated type of each projection of a `SELECT`, past its alias. +pub(crate) fn projection_types(expression: &Expression) -> Vec> { + let Expression::Select(select) = expression else { + return Vec::new(); + }; + let mut types: Vec> = Vec::with_capacity(select.expressions.len()); + for projection in &select.expressions { + let value: &Expression = match projection { + Expression::Alias(alias) => &alias.this, + other => other, + }; + types.push(value.inferred_type().cloned()); + } + types +} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs index eabbb7c0e9..45e65c7c99 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/annotated_output_types.rs @@ -1,19 +1,19 @@ use std::collections::HashMap; use polyglot_sql::expressions::DataType; -use polyglot_sql::optimizer::annotate_types::function_result_is_known; -use polyglot_sql::traversal::ExpressionWalk; use polyglot_sql::{DialectType, Expression, ParseOptions, ValidationSchema}; +use crate::query_analysis::annotated_types::{has_untyped_call, projection_types}; use crate::query_analysis::compatibility_types::{NULL_TYPE, infer_outputs}; +use crate::query_analysis::types::AnnotatedOutput; -/// The output types of the one query in `sql` by full annotation; `None` if a call is untyped. +/// Each output of the one query in `sql`, typed by full annotation; `None` if a call is untyped. #[must_use] pub fn annotated_output_types( sql: &str, dialect: &str, schema: &ValidationSchema, -) -> Option)>> { +) -> Option> { let dialect: DialectType = match dialect.parse() { Ok(dialect) => dialect, Err(_) => return None, @@ -38,34 +38,18 @@ pub fn annotated_output_types( if has_untyped_call(&expression, dialect) { return None; } - let mut outputs: Vec<(String, Option)> = Vec::new(); - for (name, data_type) in infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) { - outputs.push((name, data_type.filter(|value| value != NULL_TYPE))); - } - Some(outputs) -} - -/// Whether a call under `node`, outside a typed marker, has an unknown or only guessed type. -fn has_untyped_call(node: &Expression, dialect: DialectType) -> bool { - match node { - Expression::Annotated(annotation) if annotation.type_assertion.is_some() => false, - Expression::Function(function) - if !function_result_is_known(function, dialect) || is_unknown(node) => - { - true - } - Expression::AggregateFunction(_) | Expression::WindowFunction(_) if is_unknown(node) => { - true - } - _ => node - .children() + let annotated: Vec> = projection_types(&expression); + let mut outputs: Vec = Vec::new(); + for (index, (name, data_type)) in + infer_outputs(&expression, Some(schema), &HashMap::new(), dialect) .into_iter() - .any(|child| has_untyped_call(child, dialect)), + .enumerate() + { + outputs.push(( + name, + data_type.filter(|value| value != NULL_TYPE), + annotated.get(index).cloned().flatten(), + )); } -} - -/// Whether annotation left `node` without a type, or with a wholly unknown one. -fn is_unknown(node: &Expression) -> bool { - node.inferred_type() - .is_none_or(|data_type| *data_type == DataType::Unknown) + Some(outputs) } diff --git a/crates/sqlbuild-analysis/src/query_analysis/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/mod.rs index 94f93ea625..847eabad9a 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/mod.rs @@ -1,3 +1,5 @@ +#[path = "_helpers/annotated_types.rs"] +mod annotated_types; #[path = "_helpers/borrowed_facts.rs"] mod borrowed_facts; #[path = "_helpers/compatibility_types.rs"] @@ -5,6 +7,7 @@ mod compatibility_types; #[path = "_helpers/engine.rs"] mod engine; pub mod main; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/query_analysis/types.rs b/crates/sqlbuild-analysis/src/query_analysis/types.rs new file mode 100644 index 0000000000..ff1b963f81 --- /dev/null +++ b/crates/sqlbuild-analysis/src/query_analysis/types.rs @@ -0,0 +1,6 @@ +//! Shapes of query analysis results. + +use polyglot_sql::expressions::DataType; + +/// One output of a fully annotated query: its name, inferred type, and annotated type. +pub type AnnotatedOutput = (String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index 1133098f65..40631e8e60 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; +use polyglot_sql::expressions::DataType; use polyglot_sql::{DialectType, SchemaValidationOptions, ValidationSchema, validate_with_schema}; use serde_json::{Value, json}; @@ -16,14 +17,16 @@ use crate::sql_macros::constants::{ }; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentShape, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, SqlMacroForm, - SqlMacroLocation, TypeFamily, + ArgumentShape, BodyType, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, + SqlMacroForm, SqlMacroLocation, TypeFamily, }; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::TypeNormalization; -/// Each body check's verdict, by the probe it ran, so a macro is typed once per process. -static VERDICTS: OnceLock>>> = OnceLock::new(); +/// Each body's type, by the probe that typed it, so a macro is typed once per process. +static BODY_TYPES: OnceLock>> = OnceLock::new(); +/// Each body's first type error, by the probe that found it. +static TYPE_ERRORS: OnceLock>>> = OnceLock::new(); /// The dialect an adapter's SQL is analysed in. pub(crate) fn adapter_dialect(adapter: &str) -> &str { @@ -97,15 +100,40 @@ fn body_probe( Some((probe, table)) } -/// The body's inferred type, when inference knows it completely. -fn inferred_body_type(probe: &str, table: Value, dialect: &str) -> Option { +/// What inference knows of the body's type. +fn inferred_body_type(probe: &str, table: Value, dialect: &str) -> BodyType { let Ok(schema) = serde_json::from_value::(json!({ "tables": [table] })) else { - return None; + return BodyType::Unknown; + }; + let Some(outputs) = annotated_output_types(probe, dialect, &schema) else { + return BodyType::Unknown; + }; + let Some((_, shown, annotated)) = outputs.into_iter().next() else { + return BodyType::Unknown; + }; + match (shown, annotated) { + (Some(shown), _) if !shown.to_ascii_uppercase().contains(UNKNOWN_TYPE_HINT) => { + BodyType::Known(shown) + } + (Some(shown), Some(data_type)) + if data_type.contains_unknown() && data_type != DataType::Unknown => + { + BodyType::PartlyKnown { shown, data_type } + } + _ => BodyType::Unknown, + } +} + +/// Whether `returns` cannot fill the unknown components of `partial` without changing its shape. +fn changes_shape(returns: &str, partial: &DataType, dialect: &str) -> bool { + let Ok(dialect_type) = dialect.parse::() else { + return false; }; - let outputs = annotated_output_types(probe, dialect, &schema)?; - let (_, data_type) = outputs.into_iter().next()?; - data_type.filter(|inferred| !inferred.to_ascii_uppercase().contains(UNKNOWN_TYPE_HINT)) + match polyglot_sql::parse_data_type(returns, dialect_type) { + Ok(declared) => !partial.is_refined_by(&declared), + Err(_) => false, + } } /// Whether body type `inferred` is outside the family of `returns`, or another non-family type. @@ -135,9 +163,13 @@ fn disagrees(returns: &str, inferred: &str, adapter: &str) -> bool { } } -/// The verdict cached under `key`, or `compute`'s verdict, cached for the rest of the process. -fn remembered(key: String, compute: impl FnOnce() -> Option) -> Option { - let verdicts = VERDICTS.get_or_init(|| Mutex::new(HashMap::new())); +/// The verdict `cache` holds under `key`, or `compute`'s verdict, cached for the process. +fn remembered( + cache: &OnceLock>>, + key: String, + compute: impl FnOnce() -> T, +) -> T { + let verdicts = cache.get_or_init(|| Mutex::new(HashMap::new())); let mut verdicts = match verdicts.lock() { Ok(verdicts) => verdicts, Err(poisoned) => poisoned.into_inner(), @@ -145,7 +177,7 @@ fn remembered(key: String, compute: impl FnOnce() -> Option) -> Option = compute(); + let verdict: T = compute(); verdicts.insert(key, verdict.clone()); verdict } @@ -167,16 +199,26 @@ pub(crate) fn check_body_returns( }; let dialect: &str = adapter_dialect(adapter); let key: String = format!("returns\n{dialect}\n{table}\n{probe}"); - let inferred: Option = remembered(key, || inferred_body_type(&probe, table, dialect)); - let Some(inferred) = inferred else { - return Ok(()); - }; - let shown: &str = if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) { - VARCHAR_TYPE - } else { - &inferred + let body_type: BodyType = remembered(&BODY_TYPES, key, || { + inferred_body_type(&probe, table, dialect) + }); + let partial: bool = matches!(body_type, BodyType::PartlyKnown { .. }); + let (shown, wrong): (String, bool) = match body_type { + BodyType::Unknown => return Ok(()), + BodyType::Known(inferred) if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) => ( + VARCHAR_TYPE.to_owned(), + disagrees(returns, VARCHAR_TYPE, adapter), + ), + BodyType::Known(inferred) => { + let wrong: bool = disagrees(returns, &inferred, adapter); + (inferred, wrong) + } + BodyType::PartlyKnown { shown, data_type } => { + let wrong: bool = changes_shape(returns, &data_type, dialect); + (shown, wrong) + } }; - if !disagrees(returns, shown, adapter) { + if !wrong { return Ok(()); } Err(SqlMacroError::new( @@ -187,9 +229,15 @@ pub(crate) fn check_body_returns( ), ) .at(location) - .helped(format!( - "declare returns {shown}, CAST the body to {returns}, or remove returns to use the inferred type" - ))) + .helped(if partial { + format!( + "declare a returns of the body's shape {shown} with its unknown parts filled in, CAST the body to {returns}, or remove returns to use the inferred type" + ) + } else { + format!( + "declare returns {shown}, CAST the body to {returns}, or remove returns to use the inferred type" + ) + })) } /// The first type error the body's probe shows, as polyglot reports it. @@ -249,7 +297,9 @@ pub(crate) fn check_body_types( }; let dialect: &str = adapter_dialect(adapter); let key: String = format!("types\n{dialect}\n{table}\n{probe}"); - let found: Option = remembered(key, || body_type_error(&probe, table, dialect)); + let found: Option = remembered(&TYPE_ERRORS, key, || { + body_type_error(&probe, table, dialect) + }); let Some(found) = found else { return Ok(()); }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 83f818406d..7906f7aad1 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -409,3 +409,17 @@ pub struct SqlMacroFinding { pub call_end: (usize, usize), pub declared_at: SqlMacroLocation, } + +/// What type inference knows of an expression macro body over its declared arguments. +#[derive(Clone, Debug)] +pub(crate) enum BodyType { + /// A completely known type. + Known(String), + /// A type with unknown components, as shown, and as annotation typed it. + PartlyKnown { + shown: String, + data_type: polyglot_sql::expressions::DataType, + }, + /// Nothing, or only a guess. + Unknown, +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index 90db424af9..db7f20208e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -203,6 +203,32 @@ expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), + SqlMacroCompileTestCase( + description="a_partly_unknown_body_of_another_shape_is_reported", + files={ + **text_macro("m", "BIGINT", "[]", argument="BIGINT"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P024]: macro m() declares returns BIGINT, but its body is UNKNOWN[]", + ), + ), + SqlMacroCompileTestCase( + description="a_returns_of_the_body_shape_fills_its_unknown_parts", + files={ + **text_macro("m", "BIGINT[]", "[]", argument="BIGINT"), + **contract_model( + "orders", + 'v (type "BIGINT[]")', + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n', + ), + }, + expected_returncode=0, + expected_fragments=("0 errors, 0 warnings",), + ), SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ From a7988bef960f79633aafc92def770ec58c51327a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:36:22 +0100 Subject: [PATCH 23/28] fix: leave a SQL macro call unmarked where a marker would break parsing --- .../_helpers/typing/returns_check.rs | 7 ++- .../sql_macros/_helpers/typing/typed_calls.rs | 62 +++++++++++++++++++ .../src/sql_macros/constants.rs | 7 +++ .../sql_macros/main/typed_analysis_pass.rs | 4 +- .../src/sql_macros/main/typed_analysis_sql.rs | 4 +- .../src/sql_macros/tests/test_typed_calls.rs | 18 +++++- .../test_sql_macro_typing_matrix.py | 14 +++++ 7 files changed, 107 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index 40631e8e60..dae9bb5d27 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; use crate::query_analysis::main::annotated_output_types::annotated_output_types; use crate::sql_macros::_helpers::parsing::body::expand_body; use crate::sql_macros::_helpers::rendering::arguments::type_family; -use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; +use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; use crate::sql_macros::constants::{ BODY_TYPE_CODE, INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, TYPE_ERROR_CODES, TYPE_WARNING_CODES, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, @@ -94,7 +94,10 @@ fn body_probe( Ok(expansion) => expansion, Err(_) => return None, }; - let (typed, _) = typed_sql(&expansion.sql, &expansion.calls); + let (typed, _) = typed_sql( + &expansion.sql, + &markable_calls(&expansion.sql, &expansion.calls), + ); let probe: String = format!("SELECT ({typed}) AS b FROM {STAND_IN_RELATION}"); let table: Value = json!({"name": STAND_IN_RELATION, "columns": columns}); Some((probe, table)) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs index 1bdf1bf058..38041808d1 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -1,9 +1,13 @@ //! Expression macro calls followed through the edits that move them, and their typed markers. +use std::borrow::Cow; + use polyglot_sql::expressions::{TYPED_EXPRESSION_FUNCTION, TYPED_LIKE_EXPRESSION_FUNCTION}; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; use crate::compiled_project::models::SplicedSpan; use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; +use crate::sql_macros::constants::MARKER_PARSE_DIALECTS; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `calls` moved `offset` bytes later. @@ -150,3 +154,61 @@ pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec(sql: &str, calls: &'c [TypedCall]) -> Cow<'c, [TypedCall]> { + if !calls.iter().any(|call| in_keyword_slot(sql, call)) { + return Cow::Borrowed(calls); + } + let Some(dialect) = MARKER_PARSE_DIALECTS + .into_iter() + .find(|dialect| parses_unmarked(sql, *dialect)) + else { + return Cow::Borrowed(calls); + }; + if parses_unmarked(&typed_sql(sql, calls).0, dialect) { + return Cow::Borrowed(calls); + } + let mut kept: Vec = Vec::with_capacity(calls.len()); + for call in calls { + kept.push(call.clone()); + let marked: bool = call.returns.is_some() || call.returns_like.is_some(); + if marked && !parses_unmarked(&typed_sql(sql, &kept).0, dialect) { + kept.pop(); + } + } + Cow::Owned(kept) +} + +/// Whether `call` may stand where only a keyword or a type can: bare words, or after `AS`. +fn in_keyword_slot(sql: &str, call: &TypedCall) -> bool { + let (Some(text), Some(before)) = (sql.get(call.start..call.end), sql.get(..call.start)) else { + return false; + }; + let bare: bool = text.chars().all(|character| { + character.is_alphanumeric() || character == '_' || character.is_whitespace() + }); + let previous: &str = before + .trim_end() + .rsplit(|character: char| !(character.is_alphanumeric() || character == '_')) + .next() + .unwrap_or_default(); + bare || previous.eq_ignore_ascii_case("AS") +} + +/// Whether `sql` parses in `dialect` with every typed marker lowered. +fn parses_unmarked(sql: &str, dialect: DialectType) -> bool { + match polyglot_sql::parse(sql, dialect) { + Ok(statements) => !statements + .iter() + .any(|statement| statement.dfs().any(is_unlowered_marker)), + Err(_) => false, + } +} + +/// Whether `node` is a typed marker the parser left as an ordinary function. +fn is_unlowered_marker(node: &Expression) -> bool { + matches!(node, Expression::Function(function) + if function.name.eq_ignore_ascii_case(TYPED_EXPRESSION_FUNCTION) + || function.name.eq_ignore_ascii_case(TYPED_LIKE_EXPRESSION_FUNCTION)) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 7c0e88879a..c1a26b2e54 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -125,3 +125,10 @@ pub(crate) const ALIAS_PRECEDING_KEYWORDS: [&str; 13] = [ "FROM", "JOIN", "ON", "AND", "OR", "NOT", "WHERE", "USING", "LATERAL", "IN", "IS", "BY", "SELECT", ]; +/// Dialects tried in turn to judge whether typed markers keep a query parseable. +pub(crate) const MARKER_PARSE_DIALECTS: [polyglot_sql::DialectType; 4] = [ + polyglot_sql::DialectType::Generic, + polyglot_sql::DialectType::DuckDB, + polyglot_sql::DialectType::Snowflake, + polyglot_sql::DialectType::BigQuery, +]; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs index 854227c5e2..52d92cd232 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs @@ -1,11 +1,11 @@ use crate::semantic_validation::models::Expansion; -use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; +use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; use crate::sql_macros::models::TypedCall; /// `sql` with its typed calls marked and the pass mapping offsets back, if any call is marked. #[must_use] pub fn typed_analysis_pass(sql: &str, calls: &[TypedCall]) -> Option<(String, Vec)> { - let (typed, insertions) = typed_sql(sql, calls); + let (typed, insertions) = typed_sql(sql, &markable_calls(sql, calls)); if insertions.is_empty() { return None; } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs index 0740dc6155..3759ba668d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs @@ -1,8 +1,8 @@ -use crate::sql_macros::_helpers::typing::typed_calls::typed_sql; +use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `sql` with its typed calls wrapped in analysis-only markers, and each marker insertion. #[must_use] pub fn typed_analysis_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { - typed_sql(sql, calls) + typed_sql(sql, &markable_calls(sql, calls)) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 1ea1261529..851a67829e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -138,9 +138,9 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns }, TypedAnalysisTestCase { description: "adjacent calls close before the next opens", - sql: "SELECT ab FROM t", - calls: &[(7, 8, Some("INT")), (8, 9, Some("INT"))], - expected_sql: "SELECT __polyglot_typed(a, 'INT')__polyglot_typed(b, 'INT') FROM t", + sql: "SELECT (a)(b) FROM t", + calls: &[(7, 10, Some("INT")), (10, 13, Some("INT"))], + expected_sql: "SELECT __polyglot_typed((a), 'INT')__polyglot_typed((b), 'INT') FROM t", }, TypedAnalysisTestCase { description: "a call ending in a line comment closes it first", @@ -148,6 +148,18 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns calls: &[(7, 16, Some("DECIMAL(18, 2)"))], expected_sql: "SELECT __polyglot_typed(x -- note\n, 'DECIMAL(18, 2)')\nFROM t", }, + TypedAnalysisTestCase { + description: "a call in a keyword slot is left unmarked", + sql: "SELECT EXTRACT(YEAR FROM t.c) AS y FROM t", + calls: &[(15, 19, Some("VARCHAR"))], + expected_sql: "SELECT EXTRACT(YEAR FROM t.c) AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a bare-word call in an expression position is marked", + sql: "SELECT YEAR AS y FROM t", + calls: &[(7, 11, Some("VARCHAR"))], + expected_sql: "SELECT __polyglot_typed(YEAR, 'VARCHAR') AS y FROM t", + }, TypedAnalysisTestCase { description: "a quote in returns is doubled", sql: "SELECT x FROM t", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index cad947b5b1..a092456475 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -378,6 +378,20 @@ def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stan expected_returncode=0, expected_fragments=_CLEAN, ), + SqlMacroCompileTestCase( + description="a_call_in_a_keyword_slot_keeps_the_analysis_whole", + files={ + **text_macro("part", "VARCHAR", "YEAR"), + **contract_model( + "orders", + "y (type BIGINT)", + f"SELECT EXTRACT(@sql.part(o.status) FROM o.created_at) AS y {_FROM}\n", + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("Semantic checks were partial",), + ), SqlMacroCompileTestCase( description="the_compiled_sql_has_no_marker", files={ From 98359d06a7c969686bc3fc6f66972a0630cc8720 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:45:42 +0100 Subject: [PATCH 24/28] fix: validate SQL macro returns types for every adapter --- .../_helpers/rendering/expansion.rs | 11 +++++-- .../src/sql_macros/_helpers/typing/mod.rs | 1 + .../_helpers/typing/return_types.rs | 21 ++++++++++++ .../sql_macros/test_sql_macro_returns.py | 32 +++++++++++++++++++ 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs index 77c4d9c290..96c8c22834 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs @@ -16,6 +16,7 @@ use crate::sql_macros::_helpers::rendering::arguments::{ use crate::sql_macros::_helpers::rendering::cte_rename::{renamed_ctes, take_cte_argument}; use crate::sql_macros::_helpers::rendering::placement::top_level_parts; use crate::sql_macros::_helpers::rendering::placement::{CallSite, cte_names, select_names}; +use crate::sql_macros::_helpers::typing::return_types::returns_type_error; use crate::sql_macros::_helpers::typing::returns_check::{check_body_returns, check_body_types}; use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; use crate::sql_macros::constants::{ @@ -637,9 +638,15 @@ fn check_adapter_types( } } if let Some(returns) = &definition.returns - && let Some(error) = invalid(returns, format!("returns of macro {}()", definition.name)) + && let Some(message) = returns_type_error( + returns, + adapter, + &format!("returns of macro {}()", definition.name), + ) { - return Err(error.at(macro_location(definition, 1, 1))); + return Err(SqlMacroError::new(INVALID_DEFINITION_CODE, message) + .helped("use a SQL type the warehouse accepts, as in schema declarations") + .at(macro_location(definition, 1, 1))); } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs index 7c872088b8..0b5e9f1273 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod probes; +pub(crate) mod return_types; pub(crate) mod returns_check; pub(crate) mod typed_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs new file mode 100644 index 0000000000..9b4c2ca91e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs @@ -0,0 +1,21 @@ +//! A SQL macro's declared `returns` type checked against the adapter's SQL analysis dialect. + +use polyglot_sql::DialectType; + +use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; + +/// Why `returns` is not a type the adapter's SQL analysis dialect parses. +pub(crate) fn returns_type_error(returns: &str, adapter: &str, context: &str) -> Option { + let dialect_name: &str = adapter_dialect(adapter); + let Ok(dialect) = dialect_name.parse::() else { + return None; + }; + let Err(error) = polyglot_sql::parse_data_type(returns, dialect) else { + return None; + }; + let reason: String = error.to_string(); + Some(format!( + "{context} type '{returns}' is not valid for adapter '{adapter}' SQL analysis dialect \ + '{dialect_name}': {reason}" + )) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index db7f20208e..bcd59932d5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -229,6 +229,38 @@ expected_returncode=0, expected_fragments=("0 errors, 0 warnings",), ), + SqlMacroCompileTestCase( + description="an_invalid_returns_is_reported_on_postgres", + files={ + **adapter_project("postgres"), + **text_macro("m", "FOO BAR", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'FOO BAR' is not valid for adapter " + "'postgres' SQL analysis dialect 'postgres'", + ), + unexpected_fragments=("__polyglot",), + ), + SqlMacroCompileTestCase( + description="an_invalid_returns_is_reported_on_motherduck", + files={ + **adapter_project("motherduck"), + **text_macro("m", "FOO BAR", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'FOO BAR' is not valid for adapter " + "'motherduck' SQL analysis dialect 'duckdb'", + ), + unexpected_fragments=("__polyglot",), + ), SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ From 59f5ea258abb5e432afcaa22f70b11b94f0ee6c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:50:13 +0100 Subject: [PATCH 25/28] fix: reject a SQL macro returns decimal the dialect cannot hold --- .../_helpers/typing/return_types.rs | 44 ++++++++++++++++--- .../sql_macros/test_sql_macro_returns.py | 26 +++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs index 9b4c2ca91e..469ac054ee 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/return_types.rs @@ -1,21 +1,55 @@ //! A SQL macro's declared `returns` type checked against the adapter's SQL analysis dialect. -use polyglot_sql::DialectType; +use polyglot_sql::expressions::DataType; +use polyglot_sql::{Dialect, DialectType, Expression}; use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; -/// Why `returns` is not a type the adapter's SQL analysis dialect parses. +/// Why `returns` is not a type the adapter's SQL analysis dialect parses and can hold. pub(crate) fn returns_type_error(returns: &str, adapter: &str, context: &str) -> Option { let dialect_name: &str = adapter_dialect(adapter); let Ok(dialect) = dialect_name.parse::() else { return None; }; - let Err(error) = polyglot_sql::parse_data_type(returns, dialect) else { - return None; + let reason: String = match polyglot_sql::parse_data_type(returns, dialect) { + Ok(parsed) => decimal_error(&parsed, dialect)?, + Err(error) => error.to_string(), }; - let reason: String = error.to_string(); Some(format!( "{context} type '{returns}' is not valid for adapter '{adapter}' SQL analysis dialect \ '{dialect_name}': {reason}" )) } + +/// Why a decimal within `data_type` has a precision or scale `dialect` cannot hold. +fn decimal_error(data_type: &DataType, dialect: DialectType) -> Option { + match data_type { + DataType::Decimal { + precision: Some(precision), + scale, + } => { + if let Some(scale) = scale.filter(|scale| scale > precision) { + return Some(format!( + "DECIMAL scale {scale} is larger than its precision {precision}" + )); + } + match Dialect::get(dialect).generate(&Expression::DataType(data_type.clone())) { + Ok(_) => None, + Err(polyglot_sql::error::Error::Unsupported { feature, .. }) => Some(feature), + Err(_) => None, + } + } + DataType::Array { element_type, .. } | DataType::List { element_type } => { + decimal_error(element_type, dialect) + } + DataType::Nullable { inner } => decimal_error(inner, dialect), + DataType::Map { + key_type, + value_type, + } => decimal_error(key_type, dialect).or_else(|| decimal_error(value_type, dialect)), + DataType::Struct { fields, .. } => fields + .iter() + .find_map(|field| decimal_error(&field.data_type, dialect)), + _ => None, + } +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py index bcd59932d5..370e4570ec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_returns.py @@ -261,6 +261,32 @@ ), unexpected_fragments=("__polyglot",), ), + SqlMacroCompileTestCase( + description="a_returns_decimal_wider_than_the_dialect_holds_is_reported", + files={ + **text_macro("m", "DECIMAL(50, 2)", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "error[P015]: returns of macro m() type 'DECIMAL(50, 2)' is not valid for adapter " + "'duckdb' SQL analysis dialect 'duckdb': DECIMAL precision 50 exceeds the target " + "range 1..=38", + ), + ), + SqlMacroCompileTestCase( + description="a_returns_decimal_scale_above_its_precision_is_reported", + files={ + **text_macro("m", "DECIMAL(4, 6)", "@value", argument="ANY"), + **orders_model( + 'SELECT @sql.m(o.amount_cents) AS v FROM __ref("stg_orders") AS o\n' + ), + }, + expected_returncode=1, + expected_fragments=("DECIMAL scale 6 is larger than its precision 4",), + ), SqlMacroCompileTestCase( description="an_untyped_call_without_returns_suggests_returns_on_k003", files={ From ce6abf4edf343432e7646fefe021c383588d46da Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:50:34 +0100 Subject: [PATCH 26/28] docs: describe shape-aware returns, returns validation and scoped __cte renaming --- .../docs/concepts/macros/sql-macros.md | 19 +++++++++++++------ .../docs/docs/concepts/macros/sql-macros.mdx | 19 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index b0b2801910..b00e5ad7b5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -149,8 +149,9 @@ SELECT * FROM final The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, `us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are -renamed with them; arguments are never renamed, even one named like a macro CTE. Later CTEs may read -the prefixed ones too. +renamed with them, wherever a name reads the macro's CTE: a table alias such as `FROM live AS l`, +or a nested `WITH` that defines its own `live`, keeps its own meaning. Arguments are never renamed, +even one named like a macro CTE. Later CTEs may read the prefixed ones too. `__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in `arguments`. A call without it keeps the macro's own names. @@ -332,8 +333,9 @@ MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); bar(LENGTH(@value), 0, 10) ``` -- The declared type applies only where the expanded SQL's type is unknown, or partly unknown, such - as an array of unknown elements. A known type stands, even a close one such as `VARCHAR(100)` for +- The declared type applies only where the expanded SQL's type is unknown. A partly unknown type, + such as `[]`, an array of unknown elements, takes `returns` only when `returns` has its shape, + such as `BIGINT[]`. A known type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. - The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each branch of a set operation, through operators and functions around the call, to contract checks @@ -348,8 +350,13 @@ bar(LENGTH(@value), 0, 10) macro (`P015`). - A fixed `returns` is checked against the body at the macro file. The body is typed from the declared argument types; a known type of another family, such as `UPPER(@value)` with - `returns BIGINT`, fails with `P024`. A body whose type cannot be inferred, for example because it - reads `ANY` arguments or calls a function of unknown type, trusts `returns`. + `returns BIGINT`, fails with `P024`, as does a partly unknown body of another shape, such as `[]` + with `returns BIGINT`. A body whose type cannot be inferred, for example because it reads `ANY` + arguments or calls a function whose type is unknown or only guessed, trusts `returns`. +- `returns` must be a type the adapter's SQL dialect accepts, including a decimal precision and + scale it can hold, such as at most `DECIMAL(38, s)` on DuckDB (`P015`). +- A call where only a keyword can stand, such as the date part of `EXTRACT`, is typed by inference + alone. - A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed by inference alone. - `returns` never changes the compiled SQL. diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index f7d0f0b30c..c87be57cd4 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -135,8 +135,9 @@ SELECT * FROM final The macro's last CTE takes exactly the `__cte` name, and its other CTEs become `__`, so this model holds `eu_raw`, `us_raw`, `eu_orders__live`, `eu_orders__ranked`, `eu_orders`, `us_orders__live`, `us_orders__ranked`, `us_orders` and `final`. References inside the body are -renamed with them; arguments are never renamed, even one named like a macro CTE. Later CTEs may read -the prefixed ones too. +renamed with them, wherever a name reads the macro's CTE: a table alias such as `FROM live AS l`, +or a nested `WITH` that defines its own `live`, keeps its own meaning. Arguments are never renamed, +even one named like a macro CTE. Later CTEs may read the prefixed ones too. `__cte` takes a bare name, is only for CTE-list calls (`P017` otherwise), and is not declared in `arguments`. A call without it keeps the macro's own names. @@ -318,8 +319,9 @@ MACRO (description "Status bar", arguments (value VARCHAR), returns VARCHAR); bar(LENGTH(@value), 0, 10) ``` -- The declared type applies only where the expanded SQL's type is unknown, or partly unknown, such - as an array of unknown elements. A known type stands, even a close one such as `VARCHAR(100)` for +- The declared type applies only where the expanded SQL's type is unknown. A partly unknown type, + such as `[]`, an array of unknown elements, takes `returns` only when `returns` has its shape, + such as `BIGINT[]`. A known type stands, even a close one such as `VARCHAR(100)` for `returns VARCHAR(5)`, and contract checks report a mismatch with `K002`. - The type flows on as an inferred type would: through CTEs, `SELECT *`, subqueries, joins and each branch of a set operation, through operators and functions around the call, to contract checks @@ -334,8 +336,13 @@ bar(LENGTH(@value), 0, 10) macro (`P015`). - A fixed `returns` is checked against the body at the macro file. The body is typed from the declared argument types; a known type of another family, such as `UPPER(@value)` with - `returns BIGINT`, fails with `P024`. A body whose type cannot be inferred, for example because it - reads `ANY` arguments or calls a function of unknown type, trusts `returns`. + `returns BIGINT`, fails with `P024`, as does a partly unknown body of another shape, such as `[]` + with `returns BIGINT`. A body whose type cannot be inferred, for example because it reads `ANY` + arguments or calls a function whose type is unknown or only guessed, trusts `returns`. +- `returns` must be a type the adapter's SQL dialect accepts, including a decimal precision and + scale it can hold, such as at most `DECIMAL(38, s)` on DuckDB (`P015`). +- A call where only a keyword can stand, such as the date part of `EXTRACT`, is typed by inference + alone. - A call inside a Python macro argument, as in `@py.bracket(@sql.status_bar(o.status))`, is typed by inference alone. - `returns` never changes the compiled SQL. From 51ae569907e4181301d9ba1ddbdd4b5f85b37e05 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:47:18 +0100 Subject: [PATCH 27/28] fix: never let a typed marker change how a SQL macro call's position parses --- .../_helpers/typing/returns_check.rs | 25 +-- .../sql_macros/_helpers/typing/typed_calls.rs | 154 ++++++++++++++---- .../src/sql_macros/constants.rs | 2 + .../src/sql_macros/main/marker_free_text.rs | 9 + .../src/sql_macros/main/mod.rs | 1 + .../sql_macros/main/typed_analysis_pass.rs | 8 +- .../src/sql_macros/main/typed_analysis_sql.rs | 8 +- .../src/sql_macros/tests/test_typed_calls.rs | 44 +++-- .../analysis_session/model_analyses.rs | 19 ++- .../_helpers/analysis_session/session.rs | 8 + .../compile_driver/assembly_models.rs | 3 +- .../compiled_views/compiled_model_analysis.rs | 22 ++- .../compiled_views/model_input_view.rs | 7 +- .../test_sql_macro_typing_matrix.py | 55 +++++++ 14 files changed, 287 insertions(+), 78 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index dae9bb5d27..0f8e0d56e2 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -10,7 +10,9 @@ use serde_json::{Value, json}; use crate::query_analysis::main::annotated_output_types::annotated_output_types; use crate::sql_macros::_helpers::parsing::body::expand_body; use crate::sql_macros::_helpers::rendering::arguments::type_family; -use crate::sql_macros::_helpers::typing::typed_calls::{markable_calls, typed_sql}; +use crate::sql_macros::_helpers::typing::typed_calls::{ + markable_calls, shifted, typed_sql, without_marker_calls, +}; use crate::sql_macros::constants::{ BODY_TYPE_CODE, INFERRED_TEXT_TYPE, RETURNS_MISMATCH_CODE, STAND_IN_RELATION, TYPE_ERROR_CODES, TYPE_WARNING_CODES, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, @@ -18,7 +20,7 @@ use crate::sql_macros::constants::{ use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ ArgumentShape, BodyType, ExpandedText, ParameterKind, RenderedArgument, SqlMacroDefinition, - SqlMacroForm, SqlMacroLocation, TypeFamily, + SqlMacroForm, SqlMacroLocation, TypeFamily, TypedCall, }; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::TypeNormalization; @@ -63,7 +65,7 @@ fn stand_in_column(index: usize) -> String { fn body_probe( definition: &SqlMacroDefinition, body: &ExpandedText, - families: bool, + (families, dialect): (bool, &str), ) -> Option<(String, Value)> { let mut arguments: Vec = Vec::with_capacity(definition.parameters.len()); let mut columns: Vec = Vec::with_capacity(definition.parameters.len()); @@ -94,11 +96,10 @@ fn body_probe( Ok(expansion) => expansion, Err(_) => return None, }; - let (typed, _) = typed_sql( - &expansion.sql, - &markable_calls(&expansion.sql, &expansion.calls), - ); - let probe: String = format!("SELECT ({typed}) AS b FROM {STAND_IN_RELATION}"); + let prefix: &str = "SELECT ("; + let unmarked: String = format!("{prefix}{}) AS b FROM {STAND_IN_RELATION}", expansion.sql); + let calls: Vec = shifted(expansion.calls, prefix.len()).collect(); + let (probe, _) = typed_sql(&unmarked, &markable_calls(&unmarked, &calls, dialect)); let table: Value = json!({"name": STAND_IN_RELATION, "columns": columns}); Some((probe, table)) } @@ -197,7 +198,8 @@ pub(crate) fn check_body_returns( let Some(returns) = definition.returns.as_deref() else { return Ok(()); }; - let Some((probe, table)) = body_probe(definition, body, false) else { + let Some((probe, table)) = body_probe(definition, body, (false, adapter_dialect(adapter))) + else { return Ok(()); }; let dialect: &str = adapter_dialect(adapter); @@ -262,7 +264,7 @@ fn body_type_error(probe: &str, table: Value, dialect: &str) -> Option { .errors .into_iter() .find(|error| is_type_code(&error.code)) - .map(|error| error.message) + .map(|error| without_marker_calls(&error.message).into_owned()) } /// Whether a polyglot validation code reports an operand or argument of the wrong type. @@ -295,7 +297,8 @@ pub(crate) fn check_body_types( if families.is_empty() { return Ok(()); } - let Some((probe, table)) = body_probe(definition, body, true) else { + let Some((probe, table)) = body_probe(definition, body, (true, adapter_dialect(adapter))) + else { return Ok(()); }; let dialect: &str = adapter_dialect(adapter); diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs index 38041808d1..f90abe1f48 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/typed_calls.rs @@ -7,7 +7,7 @@ use polyglot_sql::{DialectType, Expression, ExpressionWalk}; use crate::compiled_project::models::SplicedSpan; use crate::sql_macros::_helpers::parsing::body::with_line_comment_closed; -use crate::sql_macros::constants::MARKER_PARSE_DIALECTS; +use crate::sql_macros::constants::{MARKER_PARSE_DIALECTS, NULL_KEYWORD}; use crate::sql_macros::models::{TextSplice, TypedCall}; /// `calls` moved `offset` bytes later. @@ -117,6 +117,9 @@ pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec inner.len() { "\n" } else { @@ -155,60 +158,139 @@ pub(crate) fn typed_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec(sql: &str, calls: &'c [TypedCall]) -> Cow<'c, [TypedCall]> { - if !calls.iter().any(|call| in_keyword_slot(sql, call)) { +/// `calls` without any whose marker would change how `sql` parses in `dialect`. +pub(crate) fn markable_calls<'c>( + sql: &str, + calls: &'c [TypedCall], + dialect: &str, +) -> Cow<'c, [TypedCall]> { + let marked: usize = calls.iter().filter(|call| is_marked(sql, call)).count(); + if marked == 0 { return Cow::Borrowed(calls); } - let Some(dialect) = MARKER_PARSE_DIALECTS - .into_iter() - .find(|dialect| parses_unmarked(sql, *dialect)) - else { - return Cow::Borrowed(calls); + let parsed_dialect: Option = match dialect.parse::() { + Ok(parsed) => Some(parsed), + Err(_) => MARKER_PARSE_DIALECTS + .into_iter() + .find(|candidate| lowered_markers(sql, *candidate).is_some()), }; - if parses_unmarked(&typed_sql(sql, calls).0, dialect) { + let Some(parse_dialect) = parsed_dialect else { + return Cow::Owned(Vec::new()); + }; + if lowered_markers(&typed_sql(sql, calls).0, parse_dialect) == Some(marked) { return Cow::Borrowed(calls); } let mut kept: Vec = Vec::with_capacity(calls.len()); + let mut kept_marked: usize = 0; for call in calls { + if !is_marked(sql, call) { + kept.push(call.clone()); + continue; + } kept.push(call.clone()); - let marked: bool = call.returns.is_some() || call.returns_like.is_some(); - if marked && !parses_unmarked(&typed_sql(sql, &kept).0, dialect) { + if lowered_markers(&typed_sql(sql, &kept).0, parse_dialect) == Some(kept_marked + 1) { + kept_marked += 1; + } else { kept.pop(); } } Cow::Owned(kept) } -/// Whether `call` may stand where only a keyword or a type can: bare words, or after `AS`. -fn in_keyword_slot(sql: &str, call: &TypedCall) -> bool { - let (Some(text), Some(before)) = (sql.get(call.start..call.end), sql.get(..call.start)) else { +/// Whether `typed_sql` wraps `call` in a marker. +fn is_marked(sql: &str, call: &TypedCall) -> bool { + (call.returns.is_some() || call.returns_like.is_some()) + && sql + .get(call.start..call.end) + .is_some_and(|text| !is_bare_word(text)) +} + +/// Whether `text` is one identifier or keyword other than `NULL`, which a marker could misplace. +fn is_bare_word(text: &str) -> bool { + let word: &str = text.trim(); + if word.eq_ignore_ascii_case(NULL_KEYWORD) { return false; - }; - let bare: bool = text.chars().all(|character| { - character.is_alphanumeric() || character == '_' || character.is_whitespace() - }); - let previous: &str = before - .trim_end() - .rsplit(|character: char| !(character.is_alphanumeric() || character == '_')) + } + let mut characters = word.chars(); + characters .next() - .unwrap_or_default(); - bare || previous.eq_ignore_ascii_case("AS") + .is_some_and(|first| first.is_alphabetic() || first == '_') + && characters.all(|character| character.is_alphanumeric() || character == '_') } -/// Whether `sql` parses in `dialect` with every typed marker lowered. -fn parses_unmarked(sql: &str, dialect: DialectType) -> bool { - match polyglot_sql::parse(sql, dialect) { - Ok(statements) => !statements - .iter() - .any(|statement| statement.dfs().any(is_unlowered_marker)), - Err(_) => false, +/// How many typed markers `sql` lowers to expression nodes in `dialect`, if it parses. +fn lowered_markers(sql: &str, dialect: DialectType) -> Option { + let statements: Vec = match polyglot_sql::parse(sql, dialect) { + Ok(statements) => statements, + Err(_) => return None, + }; + let mut lowered: usize = 0; + for statement in &statements { + lowered += statement + .dfs() + .filter(|node| is_lowered_marker(node)) + .count(); } + Some(lowered) } -/// Whether `node` is a typed marker the parser left as an ordinary function. -fn is_unlowered_marker(node: &Expression) -> bool { - matches!(node, Expression::Function(function) - if function.name.eq_ignore_ascii_case(TYPED_EXPRESSION_FUNCTION) - || function.name.eq_ignore_ascii_case(TYPED_LIKE_EXPRESSION_FUNCTION)) +/// Whether `node` is a typed marker lowered to its expression node. +fn is_lowered_marker(node: &Expression) -> bool { + matches!(node, Expression::Annotated(annotated) + if annotated.type_assertion.is_some() || annotated.type_like.is_some()) +} + +/// `text` with each typed marker call, as written in a message, replaced by its first argument. +pub(crate) fn without_marker_calls(text: &str) -> Cow<'_, str> { + let lowered: String = text.to_ascii_lowercase(); + if !lowered.contains(TYPED_EXPRESSION_FUNCTION) { + return Cow::Borrowed(text); + } + let mut current: String = text.to_owned(); + while let Some(start) = current.to_ascii_lowercase().find(TYPED_EXPRESSION_FUNCTION) { + let replaced: Option = marker_argument(¤t, start) + .map(|(argument, end)| format!("{}{argument}{}", ¤t[..start], ¤t[end..])); + current = match replaced { + Some(replaced) => replaced, + None => format!( + "{}expression{}", + ¤t[..start], + ¤t[start + TYPED_EXPRESSION_FUNCTION.len()..] + ), + }; + } + Cow::Owned(current) +} + +/// The first argument of the marker call at byte `start` of `text`, and the byte after the call. +fn marker_argument(text: &str, start: usize) -> Option<(String, usize)> { + let open: usize = start + text[start..].find('(')?; + let name: &str = &text[start..open]; + if !name.eq_ignore_ascii_case(TYPED_EXPRESSION_FUNCTION) + && !name.eq_ignore_ascii_case(TYPED_LIKE_EXPRESSION_FUNCTION) + { + return None; + } + let mut depth: usize = 0; + let mut quote: Option = None; + let mut comma: Option = None; + for (offset, character) in text[open..].char_indices() { + let position: usize = open + offset; + match (quote, character) { + (Some(open_quote), _) if character == open_quote => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(') => depth += 1, + (None, ',') if depth == 1 && comma.is_none() => comma = Some(position), + (None, ')') => { + depth -= 1; + if depth == 0 { + let argument_end: usize = comma.unwrap_or(position); + return Some((text[open + 1..argument_end].trim().to_owned(), position + 1)); + } + } + _ => {} + } + } + None } diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index c1a26b2e54..f63bb88d55 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -132,3 +132,5 @@ pub(crate) const MARKER_PARSE_DIALECTS: [polyglot_sql::DialectType; 4] = [ polyglot_sql::DialectType::Snowflake, polyglot_sql::DialectType::BigQuery, ]; +/// The literal a typed marker still types, though it is one bare word. +pub(crate) const NULL_KEYWORD: &str = "NULL"; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs b/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs new file mode 100644 index 0000000000..43e8bc552d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/marker_free_text.rs @@ -0,0 +1,9 @@ +use std::borrow::Cow; + +use crate::sql_macros::_helpers::typing::typed_calls::without_marker_calls; + +/// `text` with each analysis-only typed marker call replaced by the expression it typed. +#[must_use] +pub fn marker_free_text(text: &str) -> Cow<'_, str> { + without_marker_calls(text) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index e9bc680a04..45b48d9884 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -3,6 +3,7 @@ pub mod byte_text_splices; pub mod check_sql_macro_arguments; pub mod declaration_visible_from_macro; pub mod expand_sql_macro_call; +pub mod marker_free_text; pub mod moved_typed_calls; pub mod parse_sql_macro; pub mod render_inline_collection; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs index 52d92cd232..ed93d049cc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_pass.rs @@ -4,8 +4,12 @@ use crate::sql_macros::models::TypedCall; /// `sql` with its typed calls marked and the pass mapping offsets back, if any call is marked. #[must_use] -pub fn typed_analysis_pass(sql: &str, calls: &[TypedCall]) -> Option<(String, Vec)> { - let (typed, insertions) = typed_sql(sql, &markable_calls(sql, calls)); +pub fn typed_analysis_pass( + sql: &str, + calls: &[TypedCall], + dialect: &str, +) -> Option<(String, Vec)> { + let (typed, insertions) = typed_sql(sql, &markable_calls(sql, calls, dialect)); if insertions.is_empty() { return None; } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs index 3759ba668d..efbfd529bf 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/typed_analysis_sql.rs @@ -3,6 +3,10 @@ use crate::sql_macros::models::{TextSplice, TypedCall}; /// `sql` with its typed calls wrapped in analysis-only markers, and each marker insertion. #[must_use] -pub fn typed_analysis_sql(sql: &str, calls: &[TypedCall]) -> (String, Vec) { - typed_sql(sql, &markable_calls(sql, calls)) +pub fn typed_analysis_sql( + sql: &str, + calls: &[TypedCall], + dialect: &str, +) -> (String, Vec) { + typed_sql(sql, &markable_calls(sql, calls, dialect)) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs index 851a67829e..d5f3821a2b 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_typed_calls.rs @@ -132,21 +132,21 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns }, TypedAnalysisTestCase { description: "nested calls with one range, inner recorded first", - sql: "SELECT x FROM t", - calls: &[(7, 8, Some("INT")), (7, 8, Some("BIGINT"))], - expected_sql: "SELECT __polyglot_typed(__polyglot_typed(x, 'INT'), 'BIGINT') FROM t", + sql: "SELECT t.x FROM t", + calls: &[(7, 10, Some("INT")), (7, 10, Some("BIGINT"))], + expected_sql: "SELECT __polyglot_typed(__polyglot_typed(t.x, 'INT'), 'BIGINT') FROM t", }, TypedAnalysisTestCase { description: "adjacent calls close before the next opens", - sql: "SELECT (a)(b) FROM t", - calls: &[(7, 10, Some("INT")), (10, 13, Some("INT"))], - expected_sql: "SELECT __polyglot_typed((a), 'INT')__polyglot_typed((b), 'INT') FROM t", + sql: "SELECT (t.a)||(t.b) FROM t", + calls: &[(7, 12, Some("INT")), (14, 19, Some("INT"))], + expected_sql: "SELECT __polyglot_typed((t.a), 'INT')||__polyglot_typed((t.b), 'INT') FROM t", }, TypedAnalysisTestCase { description: "a call ending in a line comment closes it first", - sql: "SELECT x -- note\nFROM t", - calls: &[(7, 16, Some("DECIMAL(18, 2)"))], - expected_sql: "SELECT __polyglot_typed(x -- note\n, 'DECIMAL(18, 2)')\nFROM t", + sql: "SELECT t.x -- note\nFROM t", + calls: &[(7, 18, Some("DECIMAL(18, 2)"))], + expected_sql: "SELECT __polyglot_typed(t.x -- note\n, 'DECIMAL(18, 2)')\nFROM t", }, TypedAnalysisTestCase { description: "a call in a keyword slot is left unmarked", @@ -155,16 +155,28 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns expected_sql: "SELECT EXTRACT(YEAR FROM t.c) AS y FROM t", }, TypedAnalysisTestCase { - description: "a bare-word call in an expression position is marked", + description: "a bare-word call is never marked", sql: "SELECT YEAR AS y FROM t", calls: &[(7, 11, Some("VARCHAR"))], - expected_sql: "SELECT __polyglot_typed(YEAR, 'VARCHAR') AS y FROM t", + expected_sql: "SELECT YEAR AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a call in a type slot is left unmarked", + sql: "SELECT CAST(t.x AS DECIMAL(18, 2)) AS y FROM t", + calls: &[(19, 33, Some("VARCHAR"))], + expected_sql: "SELECT CAST(t.x AS DECIMAL(18, 2)) AS y FROM t", + }, + TypedAnalysisTestCase { + description: "a call whose marker would not parse is left unmarked", + sql: "SELECT (t.a)(t.b) FROM t", + calls: &[(7, 12, Some("INT")), (12, 17, Some("INT"))], + expected_sql: "SELECT (t.a)(t.b) FROM t", }, TypedAnalysisTestCase { description: "a quote in returns is doubled", - sql: "SELECT x FROM t", - calls: &[(7, 8, Some("ENUM('a')"))], - expected_sql: "SELECT __polyglot_typed(x, 'ENUM(''a'')') FROM t", + sql: "SELECT t.x FROM t", + calls: &[(7, 10, Some("ENUM('a')"))], + expected_sql: "SELECT __polyglot_typed(t.x, 'ENUM(''a'')') FROM t", }, ]; for test_case in test_cases { @@ -179,7 +191,7 @@ fn given_typed_calls_when_building_the_analysis_text_then_each_call_with_returns returns: returns.map(str::to_owned), }) .collect(); - let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls); + let (typed, insertions) = typed_analysis_sql(test_case.sql, &calls, "duckdb"); assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); let removed: String = { let mut text: String = String::new(); @@ -261,7 +273,7 @@ fn given_a_returns_type_of_call_when_building_the_analysis_text_then_it_asserts_ let expanded: ExpandedText = expanded_text_for("duckdb", test_case.macros, test_case.sql).expect("expands"); - let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls); + let (typed, _) = typed_analysis_sql(&expanded.sql, &expanded.calls, "duckdb"); assert_eq!(typed, test_case.expected_sql, "{}", test_case.description); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index e1d648e057..9d387684be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -132,7 +132,11 @@ fn session_request<'py>( complete_schemas: shapes.complete, catalog_schemas: shapes.schemas, dynamic_families_by_table: families, - models: model_rows(profile.py(), &request.model_inputs)?, + models: model_rows( + profile.py(), + &request.model_inputs, + &profile_dialect(profile)?, + )?, }) } @@ -323,15 +327,22 @@ fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<( } /// Each model's native request row, in analysis order. -fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyResult> { +fn model_rows<'py>( + py: Python<'py>, + model_inputs: &[Bound<'py, PyAny>], + dialect: &str, +) -> PyResult> { let enforced: Bound<'py, PyAny> = enum_member(py, RecordId::ContractPolicy, "ENFORCED")?; let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; let mut rows: Vec = Vec::with_capacity(model_inputs.len()); for model_input in model_inputs { let values: Bound<'py, PyAny> = native_field(model_input, "config")?.getattr("values")?; let get = |key: &str| mapping_get(&values, key); - let query_sql: String = - analysis_cursor_sql(py, &input_analysis_sql(model_input)?, &get("cursor_type")?)?; + let query_sql: String = analysis_cursor_sql( + py, + &input_analysis_sql(model_input, dialect)?, + &get("cursor_type")?, + )?; let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 22da623157..c2487bf4a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -186,6 +186,8 @@ pub(crate) struct NativeModelAnalysisSession { cache_path: Option, cache_stats: Option, rule_failure: RuleFailure, + /// The analysis dialect the session's models were marked and analysed in. + dialect: String, } /// What an internal native failure of the session names. @@ -208,6 +210,10 @@ impl NativeModelAnalysisSession { } /// The running session, or `NativeCompilerError` once it has failed or finished. + pub(crate) fn dialect(&self) -> &str { + &self.dialect + } + fn running(&mut self) -> PyResult { self.inner .take() @@ -347,6 +353,7 @@ pub(crate) fn start_model_analysis_session( request, ))?; let rule_failure: RuleFailure = RuleFailure::default(); + let dialect: String = request.dialect.clone(); request.nullability_callback = adapter_rules .map(|(rules, nullability)| nullability_callback(rules, nullability, rule_failure.clone())); let catalog = &catalog.inner; @@ -381,6 +388,7 @@ pub(crate) fn start_model_analysis_session( cache_path, cache_stats, rule_failure, + dialect, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 3f0af799b1..a5b9e0a289 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; +use crate::bindings::_helpers::analysis_session::profile_inputs::profile_dialect; use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; @@ -81,7 +82,7 @@ fn compiled_model<'py>( py, &analysis_cursor_sql( py, - &input_analysis_sql(model_input)?, + &input_analysis_sql(model_input, &profile_dialect(&context.profile)?)?, &mapping_get(&config_values, "cursor_type")?, )?, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs index cc85d0e41d..c86fbe3160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/compiled_model_analysis.rs @@ -18,6 +18,7 @@ use sqlbuild_analysis::binding_locations::models::{ }; use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use sqlbuild_analysis::sql_macros::constants::{ARGUMENT_CODE, SUPERSEDED_ANALYSIS_CODE}; +use sqlbuild_analysis::sql_macros::main::marker_free_text::marker_free_text; use sqlbuild_analysis::sql_macros::main::typed_analysis_pass::typed_analysis_pass; use sqlbuild_analysis::sql_macros::models::SqlMacroFinding; @@ -145,7 +146,8 @@ pub(crate) fn analysis_record( if let PivotOutcome::Proof(proof) = contract { record.proof = Some(proof.clone()); } - record.binding = with_sql_macro_findings(located(input, outcome)?, outcome); + record.binding = + with_sql_macro_findings(located(input, outcome, session.dialect())?, outcome); record.binding_validated = analysed.binding_validated; } if let Some(proof) = record.proof.as_ref().filter(|proof| proof.output_proven) { @@ -156,7 +158,11 @@ pub(crate) fn analysis_record( } /// The model's binding diagnostics at their authored locations. -fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> { +fn located( + input: &ModelInputRecord, + outcome: &ModelOutcome, + dialect: &str, +) -> PyResult> { let rows: &[DiagnosticRow] = &outcome.analysis.binding_diagnostics; if rows.is_empty() { return Ok(Vec::new()); @@ -168,7 +174,7 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult = None; let typed: Option<(String, Vec)> = if *expansion.expanded_sql == *input.query_sql { - typed_analysis_pass(&expansion.expanded_sql, &input.typed_calls) + typed_analysis_pass(&expansion.expanded_sql, &input.typed_calls, dialect) } else { None }; @@ -199,7 +205,15 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult = + locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err)?; + for diagnostic in &mut located { + diagnostic.message = marker_free_text(&diagnostic.message).into_owned(); + if let Some(help) = diagnostic.help.as_mut() { + *help = marker_free_text(help).into_owned(); + } + } + Ok(located) } /// The SQL macro findings, then the binding diagnostics without unknown columns inside a call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs index 4b39a8473e..2c801e2ef9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_input_view.rs @@ -562,10 +562,13 @@ pub(crate) fn input_sql_macro_checks( } /// The analysis text of `model_input`: its compiled query with each typed SQL macro call marked. -pub(crate) fn input_analysis_sql(model_input: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn input_analysis_sql( + model_input: &Bound<'_, PyAny>, + dialect: &str, +) -> PyResult { if let Ok(native) = model_input.cast::() { return native.get().read(model_input.py(), |record, _| { - Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls).0) + Ok(typed_analysis_sql(&record.query_sql, &record.typed_calls, dialect).0) }); } native_field(model_input, "query_sql")?.extract() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index a092456475..32bd9cb50d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -13,6 +13,7 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( BRACKET, + adapter_project, contract_model, inspected_text, lineage_edges, @@ -392,6 +393,60 @@ def test_given_a_known_different_body_when_compiling_then_the_inferred_type_stan expected_fragments=_CLEAN, unexpected_fragments=("Semantic checks were partial",), ), + SqlMacroCompileTestCase( + description="a_call_as_a_snowflake_dateadd_part", + files={ + **adapter_project("snowflake"), + **text_macro("p", "VARCHAR", "day"), + **orders_model(f"SELECT DATEADD(@sql.p(o.status), 1, o.created_at) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_snowflake_datediff_part", + files={ + **adapter_project("snowflake"), + **text_macro("p", "VARCHAR", "day"), + **orders_model( + f"SELECT DATEDIFF(@sql.p(o.status), o.created_at, o.updated_at) AS y {_FROM}\n" + ), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_an_interval_unit", + files={ + **text_macro("p", "VARCHAR", "DAY"), + **orders_model(f"SELECT o.created_at + INTERVAL 1 @sql.p(o.status) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_bare_cast_type", + files={ + **text_macro("t", "VARCHAR", "BIGINT"), + **orders_model(f"SELECT CAST(o.order_id AS @sql.t(o.status)) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), + SqlMacroCompileTestCase( + description="a_call_as_a_parameterised_cast_type", + files={ + **text_macro("t", "VARCHAR", "DECIMAL(18, 2)"), + **orders_model(f"SELECT CAST(o.order_id AS @sql.t(o.status)) AS y {_FROM}\n"), + }, + expected_returncode=0, + expected_fragments=_CLEAN, + unexpected_fragments=("__polyglot", "Semantic checks were partial"), + ), SqlMacroCompileTestCase( description="the_compiled_sql_has_no_marker", files={ From b7606b541cd7fab23be068b24739bdfd93ab1a3b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 24 Oct 2026 14:56:14 +0100 Subject: [PATCH 28/28] fix: resolve __cte references in parenthesised joins and CTEs that read the name they shadow --- .../_helpers/rendering/cte_rename.rs | 67 ++++++++++++++----- .../sql_macros/test_sql_macro_cte_names.py | 24 +++++++ 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs index daf11aec09..709fcb84c9 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/cte_rename.rs @@ -194,16 +194,24 @@ impl<'a> CteReferences<'a> { }; if let Some(with) = with { let names: Vec = with.ctes.iter().map(|cte| cte.alias.name.clone()).collect(); + for (index, cte) in with.ctes.iter().enumerate() { + let visible: usize = if with.recursive { index + 1 } else { index }; + self.frames.push((names[..visible].to_vec(), top)); + self.visit(&cte.this, false); + self.frames.pop(); + } self.frames.push((names, top)); } let scoped: bool = if let Expression::Select(select) = node { - let sources: Vec = select + let mut sources: Vec = Vec::new(); + for source in select .from .iter() .flat_map(|from| &from.expressions) .chain(select.joins.iter().map(|join| &join.this)) - .filter_map(|source| self.scope_source(source)) - .collect(); + { + sources.extend(self.scope_sources(source)); + } self.scopes.push(sources); true } else { @@ -216,7 +224,9 @@ impl<'a> CteReferences<'a> { _ => {} } for child in node.children() { - self.visit(child, false); + if !is_cte_body(with, child) { + self.visit(child, false); + } } if scoped { self.scopes.pop(); @@ -238,25 +248,33 @@ impl<'a> CteReferences<'a> { .is_some_and(|(_, body)| *body) } - /// The name a FROM or JOIN item is read as, and whether it is an unaliased body CTE. - fn scope_source(&self, source: &Expression) -> Option { + /// Each name a FROM or JOIN item is read as, and whether it is an unaliased body CTE. + fn scope_sources(&self, source: &Expression) -> Vec { + let aliased = |alias: &Identifier| ScopeSource { + name: alias.name.clone(), + body_cte: false, + }; match source { - Expression::Table(table) => Some(match &table.alias { - Some(alias) => ScopeSource { - name: alias.name.clone(), - body_cte: false, - }, + Expression::Table(table) => vec![match &table.alias { + Some(alias) => aliased(alias), None => ScopeSource { name: table.name.name.clone(), body_cte: self.is_body_cte(table), }, - }), - Expression::Subquery(subquery) => subquery.alias.as_ref().map(|alias| ScopeSource { - name: alias.name.clone(), - body_cte: false, - }), - Expression::Paren(paren) => self.scope_source(&paren.this), - _ => None, + }], + Expression::Subquery(subquery) => subquery.alias.iter().map(aliased).collect(), + Expression::JoinedTable(joined) => match &joined.alias { + Some(alias) => vec![aliased(alias)], + None => { + let mut sources: Vec = self.scope_sources(&joined.left); + for join in &joined.joins { + sources.extend(self.scope_sources(&join.this)); + } + sources + } + }, + Expression::Paren(paren) => self.scope_sources(&paren.this), + _ => Vec::new(), } } @@ -276,6 +294,19 @@ impl<'a> CteReferences<'a> { } } +/// Whether `child` is the body of one of `with`'s CTEs, which are visited in their own frames. +fn is_cte_body(with: Option<&polyglot_sql::expressions::With>, child: &Expression) -> bool { + let Some(with) = with else { + return false; + }; + for cte in &with.ctes { + if std::ptr::eq(&cte.this, child) { + return true; + } + } + false +} + /// Whether `names` holds `name`, ignoring case. fn includes_name(names: &[String], name: &str) -> bool { names diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py index e00d6a2824..e91aafcafe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_cte_names.py @@ -123,6 +123,30 @@ query="SELECT order_id FROM orders", expected_rows=[(2,)], ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_parenthesised_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM (live JOIN @src AS s ON live.order_id = s.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_cte_read_in_a_nested_parenthesised_join", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (SELECT live.order_id FROM ((live JOIN @src AS s ON live.order_id = s.order_id) JOIN @src AS t ON t.order_id = live.order_id))" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), + SqlMacroBuildTestCase( + description="a_nested_cte_reading_the_cte_it_shadows", + files=picked_orders( + "WITH live AS (SELECT * FROM @src WHERE order_id = 2),\nlatest AS (WITH live AS (SELECT * FROM live WHERE order_id > 0) SELECT order_id FROM live)" + ), + query="SELECT order_id FROM orders", + expected_rows=[(2,)], + ), ], ids=lambda case: case.description, )