From 162f3f8c20b2e5f128ca923356f5d0836e04e3b8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:36:07 +0000 Subject: [PATCH 1/4] feat(sql-macros)!: relation slots with COLUMNS groups, replacing COLUMN OF A RELATION type declares everything a macro reads from it as slots: typed single columns and untyped COLUMNS groups the caller fills. Bodies read a slot as alias.@rel.slot, or plainly as @rel.slot: a plain group read always stands as bare column names, and a plain single-slot read always reads the slot's column through the passed relation. A plain group read inside a subquery fails when a relation of that subquery has one of its columns, or its columns are unknown, since the bare names could bind there. Calls map slots with rel.slot := column or rel.slot := (columns). Group positions, mappings, overlaps, types and name clashes are checked with exact messages. BREAKING CHANGE: COLUMN OF SQL macro arguments are removed; use relation slots. --- .../_helpers/sql_macro_parts.rs | 10 +- .../src/sql_macros/_helpers/parsing/body.rs | 432 +++++++++++++- .../sql_macros/_helpers/parsing/captures.rs | 233 ++++++++ .../sql_macros/_helpers/parsing/definition.rs | 29 +- .../src/sql_macros/_helpers/parsing/header.rs | 49 +- .../_helpers/parsing/literal_positions.rs | 4 +- .../src/sql_macros/_helpers/parsing/mod.rs | 2 + .../src/sql_macros/_helpers/parsing/slots.rs | 298 ++++++++++ .../_helpers/rendering/arguments.rs | 41 +- .../_helpers/rendering/expansion.rs | 71 ++- .../src/sql_macros/_helpers/rendering/mod.rs | 1 + .../_helpers/rendering/slot_mappings.rs | 253 +++++++++ .../src/sql_macros/_helpers/typing/probes.rs | 174 +++++- .../_helpers/typing/returns_check.rs | 15 +- .../src/sql_macros/constants.rs | 10 +- .../src/sql_macros/models.rs | 78 ++- .../sqlbuild-analysis/src/sql_macros/tests.rs | 2 + .../src/sql_macros/tests/test_definitions.rs | 80 ++- .../src/sql_macros/tests/test_expansion.rs | 8 +- .../sql_macros/tests/test_relation_slots.rs | 295 ++++++++++ .../src/dsl_headers/tests/test_dsl_headers.rs | 4 +- .../main/compile/sql_macros/_test_types.py | 22 + .../main/compile/sql_macros/helpers.py | 246 +++++++- .../sql_macros/test_sql_macro_build.py | 4 +- .../sql_macros/test_sql_macro_errors.py | 12 +- .../test_sql_macro_relation_slots.py | 533 ++++++++++++++++++ .../sql_macros/test_sql_macro_slot_scopes.py | 331 +++++++++++ .../cli/commands/main/test/test_since.py | 96 ++++ 28 files changed, 3160 insertions(+), 173 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/captures.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/slots.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/slot_mappings.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_relation_slots.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_slot_scopes.py diff --git a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs index 18a35b2a60..999824e3b5 100644 --- a/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs +++ b/crates/sqlbuild-analysis/src/definition_changes/_helpers/sql_macro_parts.rs @@ -16,8 +16,7 @@ pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String { format!("collection {set} {}", type_text(data_type)) } ParameterKind::Enum { name } => format!("enum {}", name.to_lowercase()), - ParameterKind::Relation { columns } => relation_text(columns), - ParameterKind::ColumnOf { relation } => format!("column of {}", relation.to_lowercase()), + ParameterKind::Relation { columns, groups } => relation_text(columns, groups), }; let default: String = parameter .default @@ -27,8 +26,8 @@ pub(crate) fn parameter_text(parameter: &SqlMacroParameter) -> String { format!("{} {kind} {default}", parameter.name.to_lowercase()) } -/// A relation parameter's required columns and their types. -fn relation_text(columns: &[(String, String)]) -> String { +/// A relation parameter's single-column slots with their types, then its `COLUMNS` groups. +fn relation_text(columns: &[(String, String)], groups: &[String]) -> String { let mut text: String = String::from("relation"); for (column, data_type) in columns { text.push_str(&format!( @@ -37,6 +36,9 @@ fn relation_text(columns: &[(String, String)]) -> String { type_text(data_type) )); } + for group in groups { + text.push_str(&format!(" {} columns", group.to_lowercase())); + } text } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs index 9a45512806..94772236a3 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/body.rs @@ -3,17 +3,22 @@ use polyglot_sql::expressions::{Identifier, Select}; use polyglot_sql::{DialectType, Expression, ExpressionWalk}; +use crate::sql_macros::_helpers::parsing::captures::{output_selects, slot_captures}; +use crate::sql_macros::_helpers::parsing::header::unquoted; use crate::sql_macros::_helpers::parsing::lexing::{Token, tokens}; use crate::sql_macros::_helpers::parsing::parameters::{ ParameterReference, alias_follows, parameter_references, placeholder, placeholder_parameter, with_placeholders, }; +use crate::sql_macros::_helpers::parsing::slots::{SlotRead, check_group_positions, slot_reads}; use crate::sql_macros::_helpers::rendering::arguments::is_atomic; use crate::sql_macros::_helpers::typing::typed_calls::{shifted, spliced}; -use crate::sql_macros::constants::{ARGUMENT_CODE, INVALID_DEFINITION_CODE}; +use crate::sql_macros::constants::{ + ARGUMENT_CODE, INVALID_DEFINITION_CODE, NAME_CLASH_CODE, SLOT_PLACEHOLDER_PREFIX, +}; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentShape, BodyExpansion, ExpandedText, ParameterKind, RenderedArgument, + ArgumentShape, BodyExpansion, ExpandedText, ParameterKind, RenderedArgument, SlotMapping, SqlMacroDefinition, SqlMacroForm, SqlMacroParameter, TextSplice, TypedCall, }; @@ -32,11 +37,11 @@ const EXPRESSION_PREFIX: &str = "SELECT "; const CTE_SUFFIX: &str = "\nSELECT 1"; /// The body parsed as SQL. -struct ParsedBody { - select: Box, } -fn parsed(form: SqlMacroForm, body: &str) -> Result { +pub(crate) fn parsed(form: SqlMacroForm, body: &str) -> Result { let text: String = match form { SqlMacroForm::Expression => format!("{EXPRESSION_PREFIX}{body}"), SqlMacroForm::CteList => format!("{body}{CTE_SUFFIX}"), @@ -131,10 +136,14 @@ fn defined_names( SqlMacroForm::ColumnList => { let mut names: Vec = Vec::new(); for projection in &select.expressions { - let name: String = match projection { - Expression::Alias(alias) => alias.alias.name.clone(), + match projection { + Expression::Alias(alias) => names.push(alias.alias.name.clone()), Expression::Column(column) => { - column_output_name(definition, column, arguments)? + names.extend(column_output_names(definition, column, arguments)?); + } + Expression::Dot(dot) if aliased_slot(definition, dot).is_some() => { + let (relation, slot) = aliased_slot(definition, dot).unwrap_or_default(); + names.extend(slot_output_names(definition, (relation, slot), arguments)); } _ => { return Err(format!( @@ -142,33 +151,76 @@ fn defined_names( shown_projection(projection) )); } - }; - names.push(name); + } } Ok(names) } } } -/// The output name of a bare column projection: a `COLUMN OF` argument's column, or the name. -fn column_output_name( +/// The output names of a bare column projection: a `COLUMNS` slot's columns, or the name. +fn column_output_names( definition: &SqlMacroDefinition, column: &polyglot_sql::expressions::Column, arguments: &[RenderedArgument], -) -> Result { - let Some(parameter) = parameter_of(definition, &column.name) else { - return Ok(column.name.name.clone()); - }; - if !matches!(parameter.kind, ParameterKind::ColumnOf { .. }) { +) -> Result, String> { + if let Some(parameter) = parameter_of(definition, &column.name) { return Err(format!( "every column of a column-list macro needs a name; add AS to @{}", parameter.name )); } - Ok(arguments + let slot: &str = &column.name.name; + let relation: Option<&str> = column + .table + .as_ref() + .and_then(|table| parameter_of(definition, table)) + .map(|parameter| parameter.name.as_str()); + Ok(match relation { + Some(relation) => slot_output_names(definition, (relation, slot), arguments), + None => vec![slot.to_owned()], + }) +} + +/// The relation and slot an aliased slot read `alias.@relation.slot` names, if `dot` is one. +fn aliased_slot<'d>( + definition: &'d SqlMacroDefinition, + dot: &'d polyglot_sql::expressions::DotAccess, +) -> Option<(&'d str, &'d str)> { + let Expression::Column(column) = &dot.this else { + return None; + }; + column.table.as_ref()?; + let parameter: &SqlMacroParameter = parameter_of(definition, &column.name)?; + parameter + .kind + .declares_slot(&dot.field.name) + .then_some((parameter.name.as_str(), dot.field.name.as_str())) +} + +/// The output names of a read of `relation.slot`: a group's mapped columns, or the slot's name. +fn slot_output_names( + definition: &SqlMacroDefinition, + (relation, slot): (&str, &str), + arguments: &[RenderedArgument], +) -> Vec { + let mapped: Option<&SlotMapping> = arguments .iter() - .find(|argument| definition.parameters[argument.parameter].name == parameter.name) - .map_or_else(|| parameter.name.clone(), |argument| argument.sql.clone())) + .find(|argument| definition.parameters[argument.parameter].name == relation) + .and_then(|argument| { + argument + .slots + .iter() + .find(|mapping| mapping.group && mapping.slot.eq_ignore_ascii_case(slot)) + }); + match mapped { + Some(mapping) => mapping + .columns + .iter() + .map(|column| unquoted(column)) + .collect(), + None => vec![slot.to_owned()], + } } fn shown_projection(projection: &Expression) -> String { @@ -227,17 +279,318 @@ fn substituted( } } +/// The mapping a call made for the slot a read names, if it made one. +fn read_mapping<'a>( + definition: &SqlMacroDefinition, + read: &SlotRead, + arguments: &'a [RenderedArgument], +) -> Option<&'a SlotMapping> { + arguments + .iter() + .find(|argument| { + definition.parameters[argument.parameter] + .name + .eq_ignore_ascii_case(&read.relation) + })? + .slots + .iter() + .find(|mapping| mapping.slot.eq_ignore_ascii_case(&read.slot)) +} + +/// A bare column `select` outputs twice: once by name and once in a group read beside it. +fn select_clash( + definition: &SqlMacroDefinition, + select: &Select, + groups: &[(&str, &SlotMapping)], +) -> Option { + let mut bare: Vec<&str> = Vec::new(); + let mut read: Vec<(&str, &str)> = Vec::new(); + for projection in &select.expressions { + match projection { + Expression::Column(column) => { + match column + .table + .as_ref() + .and_then(|table| parameter_of(definition, table)) + { + Some(parameter) => read.push((parameter.name.as_str(), &column.name.name)), + None => bare.push(&column.name.name), + } + } + Expression::Dot(dot) => read.extend(aliased_slot(definition, dot)), + _ => {} + } + } + for (relation, mapping) in groups { + let in_select: bool = read.iter().any(|(read_relation, slot)| { + read_relation.eq_ignore_ascii_case(relation) && slot.eq_ignore_ascii_case(&mapping.slot) + }); + if !in_select { + continue; + } + for column in &mapping.columns { + let column: String = unquoted(column); + if let Some(name) = bare.iter().find(|name| name.eq_ignore_ascii_case(&column)) { + return Some((*name).to_owned()); + } + } + } + None +} + +/// Rejects a grouped column named like a column the macro's body defines. +fn check_group_clashes( + definition: &SqlMacroDefinition, + select: &Select, + arguments: &[RenderedArgument], +) -> Result<(), SqlMacroError> { + let mut groups: Vec<(&str, &SlotMapping)> = Vec::new(); + for argument in arguments { + let relation: &str = &definition.parameters[argument.parameter].name; + for mapping in &argument.slots { + if mapping.group { + groups.push((relation, mapping)); + } + } + } + if groups.is_empty() { + return Ok(()); + } + let root: Expression = Expression::Select(Box::new(select.clone())); + let mut defined: Vec = Vec::new(); + for node in root.dfs() { + if let Expression::Select(inner) = node { + for projection in &inner.expressions { + if let Expression::Alias(alias) = projection { + defined.push(alias.alias.name.clone()); + } + } + } + } + for select in output_selects(&root) { + if let Some(clash) = select_clash(definition, select, &groups) { + defined.push(clash); + } + } + for (relation, mapping) in groups { + for column in &mapping.columns { + let bare: String = unquoted(column); + if let Some(clash) = defined + .iter() + .find(|defined| defined.eq_ignore_ascii_case(&bare)) + { + return Err(SqlMacroError::new( + NAME_CLASH_CODE, + format!( + "column {column} of slot {relation}.{} clashes with column {clash} that macro {}() defines", + mapping.slot, definition.name + ), + ) + .helped(format!( + "map {relation}.{} to columns the macro does not define, or rename {clash} in {}", + mapping.slot, definition.path + ))); + } + } + } + Ok(()) +} + +/// `body` with each of `marked` as its slot placeholder, by index, and other references as theirs. +pub(crate) fn marked_text( + (body, references): (&str, &[ParameterReference]), + marked: &[&SlotRead], +) -> String { + let mut text: String = String::with_capacity(body.len()); + let mut cursor: usize = 0; + for (position, reference) in references.iter().enumerate() { + if let Some(index) = marked.iter().position(|read| read.reference == position) { + let read: &SlotRead = marked[index]; + text.push_str(&body[cursor..read.start]); + text.push_str(&slot_placeholder(index)); + cursor = read.end; + continue; + } + if reference.start < cursor { + continue; + } + text.push_str(&body[cursor..reference.start]); + text.push_str(&placeholder(&reference.name)); + cursor = reference.end; + } + text.push_str(&body[cursor..]); + text +} + +/// Rejects a mapped single-column slot read in a select-list item without `AS`. +fn check_mapped_reads( + definition: &SqlMacroDefinition, + (body, references, reads): (&str, &[ParameterReference], &[SlotRead]), + arguments: &[RenderedArgument], +) -> Result<(), SqlMacroError> { + let mut mapped: Vec<(&SlotRead, &SlotMapping)> = Vec::new(); + for read in reads.iter().filter(|read| !read.group) { + if let Some(mapping) = read_mapping(definition, read, arguments) { + mapped.push((read, mapping)); + } + } + if mapped.is_empty() { + return Ok(()); + } + let marked: Vec<&SlotRead> = mapped.iter().map(|(read, _)| *read).collect(); + let text: String = marked_text((body, references), &marked); + let Ok(parsed_text) = parsed(definition.form, &text) else { + return Ok(()); + }; + let root: Expression = Expression::Select(parsed_text.select); + let wrapped: bool = definition.form == SqlMacroForm::Expression; + for (position, select) in output_selects(&root).into_iter().enumerate() { + if wrapped && position == 0 { + continue; + } + for projection in &select.expressions { + if matches!(projection, Expression::Alias(_)) { + continue; + } + let found: Option = projection + .dfs() + .find_map(slot_placeholder_name) + .and_then(|name| (0..mapped.len()).find(|index| name == slot_placeholder(*index))); + if let Some(index) = found { + let (read, mapping) = mapped[index]; + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "slot {}.{} of macro {}() is mapped to {}, but the macro reads it in a select list without AS, so the output column's name would depend on the mapping", + read.relation, read.slot, definition.name, mapping.columns[0] + ), + ) + .helped(format!( + "add AS to {} in {}; a mapped single-column slot needs an explicit output name", + &body[read.start..read.end], + definition.path + ))); + } + } + } + Ok(()) +} + +/// The word the `index`th mapped single-column slot read stands as while the body is parsed. +pub(crate) fn slot_placeholder(index: usize) -> String { + format!("{SLOT_PLACEHOLDER_PREFIX}{index}") +} + +/// The placeholder column name a single-column slot read stands as, if `node` is one. +pub(crate) fn slot_placeholder_name(node: &Expression) -> Option<&str> { + let Expression::Column(column) = node else { + return None; + }; + let name: &str = &column.name.name; + (column.table.is_none() && name.starts_with(SLOT_PLACEHOLDER_PREFIX)).then_some(name) +} + +/// A slot read's columns: aliased, bare for a group or in `USING`, else through the relation. +fn substituted_read( + read: &SlotRead, + argument: &RenderedArgument, + (body, qualifier): (&str, &str), + outer: Option<&ParameterKind>, +) -> String { + let mapped: Option<&SlotMapping> = argument + .slots + .iter() + .find(|mapping| mapping.slot.eq_ignore_ascii_case(&read.slot)); + let columns: Vec<&str> = match mapped { + Some(mapping) => mapping.columns.iter().map(String::as_str).collect(), + None => vec![read.written.as_str()], + }; + let prefix: Option<&str> = read.prefix.map(|(start, end)| &body[start..end]); + let forwarded: bool = matches!(argument.shape, ArgumentShape::ParameterReference { .. }); + let bare: bool = read.group || read.in_using; + let mut texts: Vec = Vec::with_capacity(columns.len()); + for column in columns { + let outer_slot: bool = + forwarded && outer.is_some_and(|kind| kind.declares_slot(&unquoted(column))); + texts.push(match prefix { + Some(prefix) if outer_slot => format!("{prefix}{}.{column}", argument.sql), + Some(prefix) => format!("{prefix}{column}"), + None if outer_slot => format!("{}.{column}", argument.sql), + None if bare => column.to_owned(), + None => format!("{qualifier}.{column}"), + }); + } + texts.join(", ") +} + +/// The outer relation's kind for an argument that forwards an outer macro's relation. +fn forwarded_kind<'o>( + argument: &RenderedArgument, + outer: Option<&'o SqlMacroDefinition>, +) -> Option<&'o ParameterKind> { + let ArgumentShape::ParameterReference { name } = &argument.shape else { + return None; + }; + outer?.parameter(name).map(|parameter| ¶meter.kind) +} + +/// Rejects a forwarded slot mapped to a slot of the outer relation of the other kind. +fn check_forwarded_kinds( + definition: &SqlMacroDefinition, + arguments: &[RenderedArgument], + outer: Option<&SqlMacroDefinition>, +) -> Result<(), SqlMacroError> { + for argument in arguments { + let ArgumentShape::ParameterReference { name } = &argument.shape else { + continue; + }; + let Some(kind) = outer + .and_then(|outer| outer.parameter(name)) + .map(|outer| &outer.kind) + else { + continue; + }; + let relation: &str = &definition.parameters[argument.parameter].name; + for mapping in &argument.slots { + for column in &mapping.columns { + let bare: String = unquoted(column); + if !kind.declares_slot(&bare) || kind.declares_group(&bare) == mapping.group { + continue; + } + let (inner, outer_kind) = if mapping.group { + ("a COLUMNS slot", "a single-column slot") + } else { + ("a single-column slot", "a COLUMNS slot") + }; + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "slot {relation}.{} of macro {}() is {inner}, but the call maps it to {name}.{bare}, {outer_kind} of the forwarded relation {name}", + mapping.slot, definition.name + ), + ) + .helped(format!( + "map {relation}.{} to slots of {name} of the same kind, or to columns of {name}", + mapping.slot + ))); + } + } + } + Ok(()) +} + /// `body` with each `@name` replaced by its argument, token by token, and the calls it holds. fn substituted_body( - definition: &SqlMacroDefinition, - (body, scanned, references): (&str, &[Token], &[ParameterReference]), + (definition, outer): (&SqlMacroDefinition, Option<&SqlMacroDefinition>), + (body, scanned): (&str, &[Token]), + (references, reads): (&[ParameterReference], &[SlotRead]), (arguments, body_calls): (&[RenderedArgument], &[TypedCall]), ) -> (String, Vec) { let mut expanded: String = String::with_capacity(body.len()); let mut splices: Vec = Vec::new(); let mut argument_calls: Vec = Vec::new(); let mut cursor: usize = 0; - for reference in references { + for (position, reference) in references.iter().enumerate() { let Some(argument) = arguments.iter().find(|argument| { definition.parameters[argument.parameter] .name @@ -246,13 +599,26 @@ fn substituted_body( continue; }; let parameter = &definition.parameters[argument.parameter]; - expanded.push_str(&body[cursor..reference.start]); + let read: Option<&SlotRead> = reads.iter().find(|read| read.reference == position); + let source: (usize, usize) = read.map_or((reference.start, reference.end), |read| { + (read.start, read.end) + }); + expanded.push_str(&body[cursor..source.0]); let text: String = substituted( ¶meter.kind, ¶meter.name, argument, (body, scanned, reference.token), ); + let text: String = match read { + Some(read) => substituted_read( + read, + argument, + (body, &text), + forwarded_kind(argument, outer), + ), + None => text, + }; let start: usize = expanded.len(); if !argument.calls.is_empty() { let inner: Option = if text == argument.sql { @@ -268,10 +634,10 @@ fn substituted_body( } expanded.push_str(&text); splices.push(TextSplice { - source: (reference.start, reference.end), + source, output: (start, expanded.len()), }); - cursor = reference.end; + cursor = source.1; } expanded.push_str(&body[cursor..]); let mut calls: Vec = argument_calls; @@ -294,7 +660,7 @@ fn within(calls: Vec, (start, end): (usize, usize)) -> Vec /// `body` with each parameter replaced by its argument, as the form inserts it at a call. pub(crate) fn expand_body( - definition: &SqlMacroDefinition, + (definition, outer): (&SqlMacroDefinition, Option<&SqlMacroDefinition>), body: &ExpandedText, arguments: &[RenderedArgument], ) -> Result { @@ -312,6 +678,9 @@ pub(crate) fn expand_body( let scanned: Vec = tokens(body).map_err(|_| invalid("it has an unclosed quote or comment".to_owned()))?; let references: Vec = parameter_references(body, &scanned); + let reads: Vec = + slot_reads(definition, (body, &scanned, &references)).map_err(|(_, error)| error)?; + check_group_positions(definition, (body, &scanned), &reads).map_err(|(_, error)| error)?; let parsed_body: ParsedBody = parsed(definition.form, &with_placeholders(body, &references)).map_err(invalid)?; if let Some(problem) = shape_problem(definition.form, &parsed_body.select) { @@ -325,9 +694,13 @@ pub(crate) fn expand_body( format!("macro {}(): {reason}", definition.name), ) })?; + check_group_clashes(definition, &parsed_body.select, arguments)?; + check_mapped_reads(definition, (body, &references, &reads), arguments)?; + check_forwarded_kinds(definition, arguments, outer)?; let (expanded, calls) = substituted_body( - definition, - (body, &scanned, &references), + (definition, outer), + (body, &scanned), + (&references, &reads), (arguments, body_calls), ); let kept: (usize, usize) = match definition.form { @@ -339,6 +712,7 @@ pub(crate) fn expand_body( names, calls: within(calls, kept), like: None, + captures: slot_captures(definition, (body, &references, &reads), arguments), }) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/captures.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/captures.rs new file mode 100644 index 0000000000..d106dde872 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/captures.rs @@ -0,0 +1,233 @@ +//! Plain `COLUMNS` slot reads inside a subquery, and the subquery relations their bare names meet. + +use polyglot_sql::expressions::Select; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::sql_macros::_helpers::parsing::body::{ + ParsedBody, marked_text, parsed, slot_placeholder, slot_placeholder_name, +}; +use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, placeholder_parameter}; +use crate::sql_macros::_helpers::parsing::slots::SlotRead; +use crate::sql_macros::models::{ArgumentShape, RenderedArgument, SlotCapture, SqlMacroDefinition}; + +/// The relations one query reads, and whether it is a subquery of an expression. +struct Scope { + sources: Vec>, + subquery: bool, +} + +/// The placeholder columns found, each with the relations of its enclosing subqueries. +#[derive(Default)] +struct Captures { + scopes: Vec, + found: Vec<(String, Vec>)>, +} + +/// The name a `FROM` or `JOIN` item reads, `@parameter` for a relation parameter. +fn source_names(source: &Expression) -> Vec> { + match source { + Expression::Table(table) => { + let name: &str = &table.name.name; + let shown: String = match placeholder_parameter(name) { + Some(parameter) => format!("@{parameter}"), + None => match &table.schema { + Some(schema) => format!("{}.{name}", schema.name), + None => name.to_owned(), + }, + }; + vec![Some(shown)] + } + Expression::JoinedTable(joined) => { + let mut names: Vec> = source_names(&joined.left); + for join in &joined.joins { + names.extend(source_names(&join.this)); + } + names + } + Expression::Paren(paren) => source_names(&paren.this), + _ => vec![None], + } +} + +/// Whether `child` of `select` is a query of its own: a CTE body, or a non-`LATERAL` `FROM` item. +fn own_scope(select: &Select, child: &Expression) -> bool { + let lateral: bool = matches!(child, Expression::Subquery(subquery) if subquery.lateral); + let source: bool = select + .from + .iter() + .flat_map(|from| &from.expressions) + .chain(select.joins.iter().map(|join| &join.this)) + .any(|known| std::ptr::eq(known, child)); + let cte: bool = select + .with + .iter() + .flat_map(|with| &with.ctes) + .any(|cte| std::ptr::eq(&cte.this, child)); + cte || (source && !lateral) +} + +/// The selects whose output names reach the query's output: all but those inside expressions. +pub(crate) fn output_selects(root: &Expression) -> Vec<&Select> { + selects_under(root, false) +} + +/// The output selects at and under `node`, which stands inside an expression if `in_expression`. +fn selects_under(node: &Expression, in_expression: bool) -> Vec<&Select> { + let mut found: Vec<&Select> = Vec::new(); + let select: Option<&Select> = match node { + Expression::Select(select) => Some(select), + _ => None, + }; + if let Some(select) = select + && !in_expression + { + found.push(select); + } + for child in node.children() { + let inside: bool = in_expression || select.is_some_and(|select| !own_scope(select, child)); + found.extend(selects_under(child, inside)); + } + found +} + +impl Captures { + fn visit(&mut self, node: &Expression, in_expression: bool) { + if let Expression::Select(select) = node { + self.select(node, select, in_expression); + return; + } + if let Some(name) = slot_placeholder_name(node) { + self.record(name); + } + for child in node.children() { + self.visit(child, in_expression); + } + } + + fn select(&mut self, node: &Expression, select: &Select, subquery: bool) { + let sources: Vec<&Expression> = select + .from + .iter() + .flat_map(|from| &from.expressions) + .chain(select.joins.iter().map(|join| &join.this)) + .collect(); + self.scopes.push(Scope { + sources: sources + .iter() + .flat_map(|source| source_names(source)) + .collect(), + subquery, + }); + for child in node.children() { + self.visit(child, !own_scope(select, child)); + } + self.scopes.pop(); + } + + /// Records a placeholder with the relations of the subqueries between it and its query. + fn record(&mut self, name: &str) { + let mut sources: Vec> = Vec::new(); + for scope in self.scopes.iter().rev() { + if !scope.subquery { + break; + } + sources.extend(scope.sources.iter().cloned()); + } + if !sources.is_empty() { + self.found.push((name.to_owned(), sources)); + } + } +} + +/// Each plain group read of `body` inside a subquery, with the relations that subquery reads. +pub(crate) fn slot_captures( + definition: &SqlMacroDefinition, + (body, references, reads): (&str, &[ParameterReference], &[SlotRead]), + arguments: &[RenderedArgument], +) -> Vec { + let argument_of = |relation: &str| { + arguments.iter().find(|argument| { + definition.parameters[argument.parameter] + .name + .eq_ignore_ascii_case(relation) + }) + }; + let marked: Vec<&SlotRead> = reads + .iter() + .filter(|read| read.group && read.prefix.is_none() && !read.in_using) + .filter(|read| { + argument_of(&read.relation).is_some_and(|argument| { + !matches!(argument.shape, ArgumentShape::ParameterReference { .. }) + }) + }) + .collect(); + if marked.is_empty() { + return Vec::new(); + } + let text: String = marked_text((body, references), &marked); + let Ok(ParsedBody { select }) = parsed(definition.form, &text) else { + return Vec::new(); + }; + let root: Expression = Expression::Select(select); + let mut captures: Captures = Captures::default(); + captures.visit(&root, false); + let mut found: Vec = Vec::new(); + for (index, read) in marked.iter().enumerate() { + let Some(argument) = argument_of(&read.relation) else { + continue; + }; + let Some(mapping) = argument + .slots + .iter() + .find(|mapping| mapping.slot.eq_ignore_ascii_case(&read.slot)) + else { + continue; + }; + let name: String = slot_placeholder(index); + let passed: &str = argument.sql.trim(); + for (_, sources) in captures.found.iter().filter(|(found, _)| *found == name) { + let mut shown: Vec> = Vec::with_capacity(sources.len()); + for source in sources { + let source: Option = source + .as_ref() + .map(|source| shown_source(source, definition, arguments)); + if source + .as_deref() + .is_some_and(|source| source.trim().eq_ignore_ascii_case(passed)) + { + continue; + } + shown.push(source); + } + if shown.is_empty() { + continue; + } + found.push(SlotCapture { + relation: read.relation.clone(), + slot: read.slot.clone(), + columns: mapping.columns.clone(), + sources: shown, + }); + } + } + found +} + +/// A source as the call reads it: a relation parameter as the relation passed for it. +fn shown_source( + source: &str, + definition: &SqlMacroDefinition, + arguments: &[RenderedArgument], +) -> String { + let Some(parameter) = source.strip_prefix('@') else { + return source.to_owned(); + }; + arguments + .iter() + .find(|argument| { + definition.parameters[argument.parameter] + .name + .eq_ignore_ascii_case(parameter) + }) + .map_or_else(|| source.to_owned(), |argument| argument.sql.clone()) +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs index c401284890..f29f195c90 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/definition.rs @@ -7,6 +7,7 @@ use crate::sql_macros::_helpers::parsing::literal_positions::{ LiteralRules, literal_error, non_literal_use, }; use crate::sql_macros::_helpers::parsing::parameters::{ParameterReference, parameter_references}; +use crate::sql_macros::_helpers::parsing::slots::{SlotRead, check_group_positions, slot_reads}; use crate::sql_macros::_helpers::rendering::arguments::{ ShapeContext, argument_shape, check_shape, }; @@ -81,6 +82,10 @@ fn checked_definition( check_parameters(&definition)?; check_parameter_references(&definition, scanned, in_body)?; let references: Vec = parameter_references(&definition.body, scanned); + let reads: Vec = slot_reads(&definition, (&definition.body, scanned, &references)) + .map_err(|(offset, error)| error.at(in_body(offset)))?; + check_group_positions(&definition, (&definition.body, scanned), &reads) + .map_err(|(offset, error)| error.at(in_body(offset)))?; if let Some(found) = non_literal_use(&definition, scanned, &references, LiteralRules::Universal) { return Err(literal_error(&definition, &found).at(in_body(found.reference.start))); @@ -217,8 +222,7 @@ fn check_returns_like( | ParameterKind::Constant { .. } | ParameterKind::Enum { .. } => Ok(()), ParameterKind::Collection { .. } - | ParameterKind::Relation { .. } - | ParameterKind::ColumnOf { .. } => Err(SqlMacroError::new( + | ParameterKind::Relation { .. } => Err(SqlMacroError::new( INVALID_DEFINITION_CODE, format!( "returns TYPE OF {} names a {} argument of macro {name}(); only a scalar argument has a type to return", @@ -286,10 +290,14 @@ fn check_parameter_references( { continue; } + let slot_mapping: bool = scanned + .get(index + 3) + .is_some_and(|assign| assign.is(body, ":=")); let qualifier: bool = scanned .get(index + 1) .is_some_and(|dot| dot.is(body, ".") && dot.start == token.end) - && !(index > 0 && scanned[index - 1].is(body, ".")); + && !(index > 0 && scanned[index - 1].is(body, ".")) + && !slot_mapping; let table: bool = index > 0 && (scanned[index - 1].is_word(body, "FROM") || scanned[index - 1].is_word(body, "JOIN")); @@ -378,7 +386,7 @@ fn check_references( Ok(()) } -/// Rejects defaults the parameter does not accept and `COLUMN OF` naming no relation argument. +/// Rejects reserved names and defaults the parameter does not accept. fn check_parameters(definition: &SqlMacroDefinition) -> Result<(), SqlMacroError> { for parameter in &definition.parameters { let location = SqlMacroLocation { @@ -400,19 +408,6 @@ fn check_parameters(definition: &SqlMacroDefinition) -> Result<(), SqlMacroError .at(location) .helped(format!("rename it; {} are built-in names", RESERVED_PARAMETER_NAMES.join(", ")))); } - if let ParameterKind::ColumnOf { relation } = ¶meter.kind { - let named: bool = definition.parameters.iter().any(|candidate| { - candidate.name.eq_ignore_ascii_case(relation) - && matches!(candidate.kind, ParameterKind::Relation { .. }) - }); - if !named { - return Err(SqlMacroError::new( - INVALID_DEFINITION_CODE, - format!("argument {} of macro {}() is COLUMN OF {relation}, but {relation} is not a RELATION argument", parameter.name, definition.name), - ) - .at(location)); - } - } if let Some(default) = ¶meter.default { check_shape( definition, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs index 3ed9989d10..81bc505363 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/header.rs @@ -4,7 +4,7 @@ use crate::sql_macros::_helpers::parsing::lexing::{ Token, TokenKind, closing_token, line_column, tokens, }; use crate::sql_macros::constants::{ - ANY_KEYWORD, ARGUMENTS_KEY, COLUMN_KEYWORD, COMMA, CONST_KEYWORD, DEFAULT_KEYWORD, + ANY_KEYWORD, ARGUMENTS_KEY, COLUMNS_KEYWORD, COMMA, CONST_KEYWORD, DEFAULT_KEYWORD, DESCRIPTION_KEY, ENUM_KEYWORD, HEADER_KEYWORD, INVALID_DEFINITION_CODE, LIST_KEYWORD, OF_KEYWORD, RAW_TYPE_NAMES, RELATION_COLUMN_MIN_TOKENS, RELATION_KEYWORD, RETURNS_KEY, SET_KEYWORD, TYPE_KEYWORD, @@ -244,7 +244,7 @@ impl HeaderParser<'_> { name_token.start, format!("argument {name} of macro {}() has no type", self.name), ) - .helped("give every argument a type, such as amount BIGINT, places INT CONST, src RELATION (id BIGINT) or key COLUMN OF src") + .helped("give every argument a type, such as amount BIGINT, places INT CONST or src RELATION (id BIGINT, keys COLUMNS)") }; let Some(first) = tokens.first() else { return Err(untyped()); @@ -258,6 +258,7 @@ impl HeaderParser<'_> { return Err(self.error(first.start, format!("argument {name} RELATION needs a parenthesised column list, such as RELATION (id BIGINT)"))); }; let mut columns: Vec<(String, String)> = Vec::new(); + let mut groups: Vec = Vec::new(); for column in split_top_level(source, &tokens[2..close]) { let column_name: &Token = &column[0]; if !matches!(column_name.kind, TokenKind::Word | TokenKind::Quoted) @@ -265,31 +266,39 @@ impl HeaderParser<'_> { { return Err(self.error( column_name.start, - format!("each column of RELATION argument {name} needs a name and a type"), + format!("each slot of RELATION argument {name} needs a name and a type or COLUMNS"), )); } - columns.push(( - unquoted(column_name.text(source)), - self.sql_type(name, &column[1..], column_name)?, - )); + let slot: String = unquoted(column_name.text(source)); + if columns + .iter() + .map(|(known, _)| known) + .chain(groups.iter()) + .any(|known| known.eq_ignore_ascii_case(&slot)) + { + return Err(self.error( + column_name.start, + format!("RELATION argument {name} declares slot {slot} twice"), + )); + } + if column[1].is_word(source, COLUMNS_KEYWORD) { + if let Some(extra) = column.get(2) { + return Err(self + .error(extra.start, format!("slot {name}.{slot} COLUMNS takes nothing after it; a COLUMNS slot is untyped and each passed column keeps its own type")) + .helped(format!("declare it {slot} COLUMNS"))); + } + groups.push(slot); + continue; + } + columns.push((slot, self.sql_type(name, &column[1..], column_name)?)); } - if columns.is_empty() { + if columns.is_empty() && groups.is_empty() { return Err(self.error( first.start, format!("RELATION argument {name} declares no columns"), )); } - return Ok(ParameterKind::Relation { columns }); - } - if first.is_word(source, COLUMN_KEYWORD) { - return match tokens { - [_, of, relation] if of.is_word(source, OF_KEYWORD) && relation.kind == TokenKind::Word => { - Ok(ParameterKind::ColumnOf { - relation: relation.text(source).to_owned(), - }) - } - _ => Err(self.error(first.start, format!("argument {name} COLUMN needs OF and a RELATION argument, such as COLUMN OF src"))), - }; + return Ok(ParameterKind::Relation { columns, groups }); } if first.is_word(source, ENUM_KEYWORD) { return match tokens { @@ -400,7 +409,7 @@ impl HeaderParser<'_> { { return Err(self .error(first.start, format!("argument {name} of macro {}() is raw text; SQL macro arguments are typed SQL", self.name)) - .helped("use a SQL type such as BIGINT, a CONST type, an ENUM, a RELATION or a COLUMN OF")); + .helped("use a SQL type such as BIGINT, a CONST type, an ENUM or a RELATION")); } Ok(type_text(source, tokens)) } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs index 839e030ed8..97c5a3164d 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/literal_positions.rs @@ -39,9 +39,7 @@ pub(crate) fn non_literal_use<'r>( let parameter = definition.parameter(&reference.name)?; if !matches!( parameter.kind, - ParameterKind::Expression { .. } - | ParameterKind::Any { .. } - | ParameterKind::ColumnOf { .. } + ParameterKind::Expression { .. } | ParameterKind::Any { .. } ) { return None; } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs index 7e98da38eb..00fcc0a884 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod body; +pub(crate) mod captures; pub(crate) mod definition; pub(crate) mod header; pub(crate) mod lexing; pub(crate) mod literal_positions; pub(crate) mod parameters; +pub(crate) mod slots; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/slots.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/slots.rs new file mode 100644 index 0000000000..f72702d0ef --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parsing/slots.rs @@ -0,0 +1,298 @@ +//! Slot reads in a SQL macro body, `@relation.slot` and `alias.@relation.slot`, and their places. + +use crate::sql_macros::_helpers::parsing::header::unquoted; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind}; +use crate::sql_macros::_helpers::parsing::parameters::ParameterReference; +use crate::sql_macros::constants::{ + GROUP_POSITIONS, INVALID_DEFINITION_CODE, PARAMETER_REFERENCE_CODE, +}; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{ParameterKind, SqlMacroDefinition}; + +/// Words after which a list item ends without being renamed, ordered or extended. +const LIST_ENDS: [&str; 18] = [ + "FROM", + "WHERE", + "GROUP", + "HAVING", + "QUALIFY", + "WINDOW", + "ORDER", + "LIMIT", + "OFFSET", + "FETCH", + "UNION", + "INTERSECT", + "EXCEPT", + "MINUS", + "ROWS", + "RANGE", + "GROUPS", + "INTO", +]; +/// Words that end the search for a list's head: the read is not in a list a group may fill. +const NOT_LIST_HEADS: [&str; 19] = [ + "FROM", + "WHERE", + "ON", + "HAVING", + "QUALIFY", + "WHEN", + "THEN", + "ELSE", + "VALUES", + "SET", + "LIMIT", + "OFFSET", + "JOIN", + "WITH", + "RETURNING", + "INTO", + "LATERAL", + "UNION", + "INTERSECT", +]; +/// The tokens one alias step takes back: a name and its dot. +const ALIAS_STEP: usize = 2; +/// The words before `BY` that open a list a group may fill. +const BY_LISTS: [&str; 3] = ["PARTITION", "GROUP", "ORDER"]; + +/// One read of a declared slot through a `RELATION` parameter. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SlotRead { + /// Byte span of the whole read, from its alias prefix, if any, to the slot's name. + pub(crate) start: usize, + pub(crate) end: usize, + /// Byte span of the alias prefix, as `s.` in `s.@weekly.order_count`. + pub(crate) prefix: Option<(usize, usize)>, + /// Index of the `@relation` reference among the body's parameter references. + pub(crate) reference: usize, + /// Token indexes of the read's first and last tokens. + pub(crate) tokens: (usize, usize), + /// The relation parameter and slot names as the macro declares them. + pub(crate) relation: String, + pub(crate) slot: String, + /// The slot's name as the body writes it. + pub(crate) written: String, + pub(crate) group: bool, + /// The read stands in `USING (...)`, unqualified, and may name an undeclared column. + pub(crate) in_using: bool, +} + +/// The list a read is a whole item of. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ListHead { + Select, + By, + Using, +} + +/// Every slot read of `body`, or why an aliased read names no slot. +pub(crate) fn slot_reads( + definition: &SqlMacroDefinition, + (body, scanned, references): (&str, &[Token], &[ParameterReference]), +) -> Result, (usize, SqlMacroError)> { + let mut reads: Vec = Vec::new(); + for (position, reference) in references.iter().enumerate() { + let Some(parameter) = definition.parameter(&reference.name) else { + continue; + }; + let ParameterKind::Relation { columns, groups } = ¶meter.kind else { + continue; + }; + let name_token: usize = reference.token; + let prefix_token: Option = prefix_start(body, scanned, name_token - 1); + let slot_token: Option<&Token> = scanned + .get(name_token + 1) + .filter(|dot| dot.is(body, ".") && dot.start == scanned[name_token].end) + .and_then(|dot| { + scanned.get(name_token + 2).filter(|word| { + matches!(word.kind, TokenKind::Word | TokenKind::Quoted) + && word.start == dot.end + }) + }); + let relation: &str = ¶meter.name; + let prefix: Option<(usize, usize)> = + prefix_token.map(|first| (scanned[first].start, reference.start)); + let shown_prefix: &str = prefix.map_or("", |(start, end)| &body[start..end]); + let Some(slot_token) = slot_token else { + if prefix.is_some() { + return Err(( + reference.start, + SqlMacroError::new( + PARAMETER_REFERENCE_CODE, + format!("{shown_prefix}@{relation} in the body of macro {}() reads RELATION argument {relation} through an alias without naming a slot", definition.name), + ) + .helped(format!("name a slot, as in {shown_prefix}@{relation}.; its slots are {}", slot_names(columns, groups))), + )); + } + continue; + }; + let written: String = slot_token.text(body).to_owned(); + let bare: String = unquoted(&written); + let declared: Option<(&str, bool)> = groups + .iter() + .find(|group| group.eq_ignore_ascii_case(&bare)) + .map(|group| (group.as_str(), true)) + .or_else(|| { + columns + .iter() + .find(|(column, _)| column.eq_ignore_ascii_case(&bare)) + .map(|(column, _)| (column.as_str(), false)) + }); + let first: usize = prefix_token.unwrap_or(name_token - 1); + let in_using: bool = + prefix.is_none() && list_head(body, scanned, first) == Some(ListHead::Using); + let declared: Option<(&str, bool)> = match declared { + None if in_using => Some((written.as_str(), false)), + declared => declared, + }; + let Some((slot, group)) = declared else { + if prefix.is_some() { + return Err(( + reference.start, + SqlMacroError::new( + PARAMETER_REFERENCE_CODE, + format!("{shown_prefix}@{relation}.{written} in the body of macro {}() names no slot of RELATION argument {relation}", definition.name), + ) + .helped(format!("its slots are {}; declare {written} in {relation} RELATION (...) or fix the name", slot_names(columns, groups))), + )); + } + continue; + }; + reads.push(SlotRead { + in_using, + start: prefix.map_or(reference.start, |(start, _)| start), + end: slot_token.end, + prefix, + reference: position, + tokens: (first, name_token + 2), + relation: relation.to_owned(), + slot: slot.to_owned(), + written, + group, + }); + } + Ok(reads) +} + +/// The first token of the alias chain that qualifies the `@` at token `at`, if one does. +pub(crate) fn prefix_start(body: &str, scanned: &[Token], at: usize) -> Option { + let mut first: Option = None; + let mut cursor: usize = at; + while cursor >= ALIAS_STEP { + let dot: &Token = &scanned[cursor - 1]; + let word: &Token = &scanned[cursor - ALIAS_STEP]; + let joined: bool = dot.is(body, ".") + && dot.end == scanned[cursor].start + && matches!(word.kind, TokenKind::Word | TokenKind::Quoted) + && word.end == dot.start; + if !joined { + break; + } + first = Some(cursor - ALIAS_STEP); + cursor -= ALIAS_STEP; + } + first +} + +fn slot_names(columns: &[(String, String)], groups: &[String]) -> String { + columns + .iter() + .map(|(column, _)| column.as_str()) + .chain(groups.iter().map(String::as_str)) + .collect::>() + .join(", ") +} + +/// Rejects a `COLUMNS` slot read anywhere but as a whole item of a list it may fill. +pub(crate) fn check_group_positions( + definition: &SqlMacroDefinition, + (body, scanned): (&str, &[Token]), + reads: &[SlotRead], +) -> Result<(), (usize, SqlMacroError)> { + for read in reads.iter().filter(|read| read.group) { + if !ends_item(body, scanned, read.tokens.1) + || list_head(body, scanned, read.tokens.0).is_none() + { + let shown: &str = &body[read.start..read.end]; + return Err(( + read.start, + SqlMacroError::new( + INVALID_DEFINITION_CODE, + format!( + "COLUMNS slot {}.{} is read as {shown} in macro {}() where a list item cannot stand; a COLUMNS slot is pasted as its columns, comma-separated, so it stands only as a whole item of {GROUP_POSITIONS}", + read.relation, read.slot, definition.name + ), + ) + .helped(format!( + "a group takes no expression, alias, ordering or indexing; read one column through a single-column slot declared as in {} RELATION (...)", + read.relation + )), + )); + } + } + Ok(()) +} + +/// Whether the read whose last token is `last` ends its list item. +fn ends_item(body: &str, scanned: &[Token], last: usize) -> bool { + match scanned.get(last + 1) { + None => true, + Some(next) if next.is(body, ",") || next.is(body, ")") || next.is(body, ";") => true, + Some(next) => LIST_ENDS.iter().any(|keyword| next.is_word(body, keyword)), + } +} + +/// The list a group may fill whose item starts at token `first`, after its head or a comma. +fn list_head(body: &str, scanned: &[Token], first: usize) -> Option { + let previous: &Token = first.checked_sub(1).map(|index| &scanned[index])?; + let starts_item: bool = previous.is(body, ",") + || previous.is(body, "(") + || previous.is_word(body, "SELECT") + || previous.is_word(body, "DISTINCT") + || previous.is_word(body, "ALL") + || previous.is_word(body, "BY"); + if !starts_item { + return None; + } + let mut depth: usize = 0; + let mut index: usize = first; + while index > 0 { + index -= 1; + let token: &Token = &scanned[index]; + if token.is(body, ")") || token.is_word(body, "END") { + depth += 1; + continue; + } + if token.is(body, "(") || token.is_word(body, "CASE") { + if depth > 0 { + depth -= 1; + continue; + } + let using: bool = + token.is(body, "(") && index > 0 && scanned[index - 1].is_word(body, "USING"); + return using.then_some(ListHead::Using); + } + if depth > 0 || token.kind != TokenKind::Word { + continue; + } + if token.is_word(body, "SELECT") { + return Some(ListHead::Select); + } + if token.is_word(body, "BY") { + let listed: bool = index > 0 + && BY_LISTS + .iter() + .any(|keyword| scanned[index - 1].is_word(body, keyword)); + return listed.then_some(ListHead::By); + } + if NOT_LIST_HEADS + .iter() + .any(|keyword| token.is_word(body, keyword)) + { + return None; + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs index 4a32f7256f..88bdb221dd 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/arguments.rs @@ -2,13 +2,14 @@ use crate::sql_macros::_helpers::parsing::header::{split_top_level, unquoted}; use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::rendering::slot_mappings::slot_mappings; use crate::sql_macros::_helpers::typing::returns_check::adapter_dialect; use crate::sql_macros::constants::{ ARGUMENT_CODE, BOOLEAN_WORDS, NAMED_ARGUMENT_MIN_TOKENS, NULL_WORD, REFERENCE_FUNCTIONS, }; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ - ArgumentShape, BoundArgument, CallArgument, ParameterKind, SqlMacroDefinition, + ArgumentShape, BoundArgument, CallArgument, ParameterKind, SlotMapping, SqlMacroDefinition, SqlMacroParameter, TypeFamily, }; use crate::type_system::main::normalize_type::normalize_type; @@ -29,10 +30,21 @@ pub(crate) fn split_arguments(arguments: &str) -> Result, SqlM let named: bool = part.len() >= NAMED_ARGUMENT_MIN_TOKENS && part[0].kind == TokenKind::Word && part[1].is(arguments, ":="); - let (name, value): (Option, &[Token]) = if named { - (Some(part[0].text(arguments).to_owned()), &part[2..]) + let slot: bool = part.len() >= NAMED_ARGUMENT_MIN_TOKENS + 2 + && part[0].kind == TokenKind::Word + && part[1].is(arguments, ".") + && matches!(part[2].kind, TokenKind::Word | TokenKind::Quoted) + && part[3].is(arguments, ":="); + let (name, slot, value): (Option, Option, &[Token]) = if named { + (Some(part[0].text(arguments).to_owned()), None, &part[2..]) + } else if slot { + ( + Some(part[0].text(arguments).to_owned()), + Some(unquoted(part[2].text(arguments))), + &part[4..], + ) } else { - (None, part) + (None, None, part) }; let (Some(first), Some(last)) = (value.first(), value.last()) else { return Err(SqlMacroError::new( @@ -42,6 +54,7 @@ pub(crate) fn split_arguments(arguments: &str) -> Result, SqlM }; found.push(CallArgument { name, + slot, text: arguments[first.start..last.end].to_owned(), offset: first.start, }); @@ -57,7 +70,13 @@ pub(crate) fn bind_arguments( let name: &str = &definition.name; let mut bound: Vec> = vec![None; definition.parameters.len()]; let mut seen_named: bool = false; + let mut slot_arguments: Vec<&CallArgument> = Vec::new(); for (position, argument) in arguments.iter().enumerate() { + if argument.slot.is_some() { + seen_named = true; + slot_arguments.push(argument); + continue; + } let index: usize = match &argument.name { Some(argument_name) => { seen_named = true; @@ -114,18 +133,22 @@ pub(crate) fn bind_arguments( text: argument.text.clone(), from_default: false, offset: Some(argument.offset), + slots: Vec::new(), }); } + let mut mappings: Vec> = slot_mappings(definition, &slot_arguments)?; let mut complete: Vec = Vec::with_capacity(bound.len()); for (index, argument) in bound.into_iter().enumerate() { let parameter: &SqlMacroParameter = &definition.parameters[index]; + let slots: Vec = std::mem::take(&mut mappings[index]); match (argument, ¶meter.default) { - (Some(argument), _) => complete.push(argument), + (Some(argument), _) => complete.push(BoundArgument { slots, ..argument }), (None, Some(default)) => complete.push(BoundArgument { parameter: index, text: default.clone(), from_default: true, offset: None, + slots, }), (None, None) => { return Err(SqlMacroError::new( @@ -142,7 +165,7 @@ pub(crate) fn bind_arguments( Ok(complete) } -fn parameter_names(definition: &SqlMacroDefinition) -> String { +pub(crate) fn parameter_names(definition: &SqlMacroDefinition) -> String { if definition.parameters.is_empty() { return "none".to_owned(); } @@ -405,10 +428,6 @@ pub(crate) fn check_shape( (ParameterKind::Relation { .. }, _) => Err(expects( "a CTE name, a table alias, or __ref(), __source() or __seed()", )), - (ParameterKind::ColumnOf { .. }, ArgumentShape::Identifier { .. }) => Ok(()), - (ParameterKind::ColumnOf { relation }, _) => { - Err(expects(&format!("a column name of {relation}"))) - } } } @@ -421,7 +440,6 @@ fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { | ParameterKind::Any { .. } | ParameterKind::Constant { .. } | ParameterKind::Enum { .. } - | ParameterKind::ColumnOf { .. } ), ParameterKind::Constant { .. } => { matches!( @@ -434,7 +452,6 @@ fn forwards(outer: &ParameterKind, inner: &ParameterKind) -> bool { matches!(outer, ParameterKind::Enum { name: outer_name } if outer_name == name) } ParameterKind::Relation { .. } => matches!(outer, ParameterKind::Relation { .. }), - ParameterKind::ColumnOf { .. } => matches!(outer, ParameterKind::ColumnOf { .. }), } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs index b9524aa6ae..d221b84b62 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/expansion.rs @@ -27,8 +27,8 @@ use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::models::{ ArgumentCheck, ArgumentExpectation, ArgumentShape, BodyExpansion, BoundArgument, CallArgument, CallPosition, CallScope, ExpandedText, InsertedNames, ParameterKind, RenderedArgument, - RenderedDeclarations, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, SqlMacroForm, - SqlMacroLocation, TextContext, TypedCall, + RenderedDeclarations, SlotCapture, SqlMacroCall, SqlMacroCheck, SqlMacroDefinition, + SqlMacroForm, SqlMacroLocation, TextContext, TypedCall, }; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; @@ -245,7 +245,11 @@ pub(crate) fn expand_call( &body, (host.adapter(), body_location(&definition)), )?; - let mut expansion: BodyExpansion = expand_body(&definition, &body, &rendered) + let outer: Option<&SqlMacroDefinition> = match context.scope { + SqlMacroScope::Macro(outer) => Some(outer.as_ref()), + SqlMacroScope::Caller => None, + }; + let mut expansion: BodyExpansion = expand_body((&definition, outer), &body, &rendered) .map_err(|error| error.at(body_location(&definition)))?; expansion.like = definition.returns_like.as_deref().and_then(|name| { rendered @@ -272,12 +276,29 @@ pub(crate) fn expand_call( (call.start, call.end), call.replaced_end, ); - if matches!(context.scope, SqlMacroScope::Caller) && context.stack.is_empty() { + let checked: bool = matches!(context.scope, SqlMacroScope::Caller) && context.stack.is_empty(); + if let Some(capture) = expansion.captures.first() + && !checked + { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "slot {}.{} is read plainly in a subquery of macro {}(), called from another macro, where its bare columns cannot be checked; write alias.@{}.{}", + capture.relation, capture.slot, definition.name, capture.relation, capture.slot + ), + ) + .at(body_location(&definition)) + .helped("qualify the read with the alias of the relation it reads") + .into()); + } + if checked { + let mut arguments: Vec = argument_checks(&definition, &bound, &rendered); + arguments.extend(capture_checks(&definition, &expansion.captures)); let check: SqlMacroCheck = SqlMacroCheck { macro_name: definition.name.clone(), scope: call_scope(context.text, scanned, call.start), plain_select: plain_select(context.text, scanned, call.start), - arguments: argument_checks(&definition, &bound, &rendered), + arguments, call: None, }; if !check.arguments.is_empty() { @@ -362,7 +383,7 @@ fn rendered_argument( calls: Vec::new(), } } - (ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. }, _) => ExpandedText { + (ParameterKind::Relation { .. }, _) => ExpandedText { sql: text.to_owned(), calls: Vec::new(), }, @@ -381,6 +402,7 @@ fn rendered_argument( sql: rendered.sql, shape, calls: rendered.calls, + slots: argument.slots.clone(), }) } @@ -619,13 +641,11 @@ fn check_adapter_types( ParameterKind::Expression { data_type } | ParameterKind::Constant { data_type } | ParameterKind::Collection { data_type, .. } => vec![data_type.as_str()], - ParameterKind::Relation { columns } => columns + ParameterKind::Relation { columns, .. } => columns .iter() .map(|(_, data_type)| data_type.as_str()) .collect(), - ParameterKind::Any { .. } - | ParameterKind::Enum { .. } - | ParameterKind::ColumnOf { .. } => Vec::new(), + ParameterKind::Any { .. } | ParameterKind::Enum { .. } => Vec::new(), }; for data_type in types { if let Some(error) = invalid( @@ -663,14 +683,10 @@ fn argument_checks( } let parameter = &definition.parameters[argument.parameter]; let expected: ArgumentExpectation = match (¶meter.kind, &rendered.shape) { - (ParameterKind::Relation { columns }, ArgumentShape::Identifier { .. }) => { + (ParameterKind::Relation { columns, .. }, ArgumentShape::Identifier { .. }) => { ArgumentExpectation::Relation { columns: columns.clone(), - } - } - (ParameterKind::ColumnOf { relation }, ArgumentShape::Identifier { .. }) => { - ArgumentExpectation::ColumnOf { - relation: relation.clone(), + slots: rendered.slots.clone(), } } ( @@ -701,6 +717,29 @@ fn argument_checks( checks } +/// One check per relation a plain group read in a subquery meets, that it holds none of its columns. +fn capture_checks(definition: &SqlMacroDefinition, captures: &[SlotCapture]) -> Vec { + let mut checks: Vec = Vec::new(); + for capture in captures { + let Some(parameter) = definition.parameter(&capture.relation) else { + continue; + }; + for source in &capture.sources { + checks.push(ArgumentCheck { + parameter: parameter.name.clone(), + text: source.clone().unwrap_or_default(), + expected: ArgumentExpectation::Capture { + slot: capture.slot.clone(), + columns: capture.columns.clone(), + source: source.clone(), + }, + declared_at: macro_location(definition, parameter.line, parameter.column), + }); + } + } + checks +} + /// The top-level query part holding the byte `offset` of `text`. fn call_scope(text: &str, scanned: &[Token], offset: usize) -> CallScope { match top_level_parts(text, scanned) { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs index 9b07958fe7..f914656264 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod arguments; pub(crate) mod cte_rename; pub(crate) mod expansion; pub(crate) mod placement; +pub(crate) mod slot_mappings; diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/slot_mappings.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/slot_mappings.rs new file mode 100644 index 0000000000..b250078eae --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/rendering/slot_mappings.rs @@ -0,0 +1,253 @@ +//! A call's `relation.slot := ...` arguments, checked and bound to the macro's declared slots. + +use crate::sql_macros::_helpers::parsing::header::{split_top_level, unquoted}; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, closing_token, tokens}; +use crate::sql_macros::_helpers::rendering::arguments::parameter_names; +use crate::sql_macros::constants::ARGUMENT_CODE; +use crate::sql_macros::errors::SqlMacroError; +use crate::sql_macros::models::{CallArgument, ParameterKind, SlotMapping, SqlMacroDefinition}; + +/// The slots each parameter's call maps, by parameter index, or why the mappings are wrong. +pub(crate) fn slot_mappings( + definition: &SqlMacroDefinition, + arguments: &[&CallArgument], +) -> Result>, SqlMacroError> { + let name: &str = &definition.name; + let mut mapped: Vec> = vec![Vec::new(); definition.parameters.len()]; + for argument in arguments { + let (Some(relation), Some(slot)) = (&argument.name, &argument.slot) else { + continue; + }; + let Some(index) = definition + .parameters + .iter() + .position(|parameter| parameter.name.eq_ignore_ascii_case(relation)) + else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("macro {name}() has no argument {relation}"), + ) + .helped(format!("its arguments are {}", parameter_names(definition)))); + }; + let parameter = &definition.parameters[index]; + let relation: &str = ¶meter.name; + let ParameterKind::Relation { columns, groups } = ¶meter.kind else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "argument {relation} of macro {name}() is {}, not a RELATION; only a RELATION argument has slots to map", + parameter.kind.label() + ), + ) + .helped(format!("pass it as {relation} := "))); + }; + let declared: Option<(String, bool)> = groups + .iter() + .find(|group| group.eq_ignore_ascii_case(slot)) + .map(|group| (group.clone(), true)) + .or_else(|| { + columns + .iter() + .find(|(column, _)| column.eq_ignore_ascii_case(slot)) + .map(|(column, _)| (column.clone(), false)) + }); + let Some((slot, group)) = declared else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("RELATION argument {relation} of macro {name}() has no slot {slot}"), + ) + .helped(format!("its slots are {}", slot_list(columns, groups)))); + }; + if mapped[index] + .iter() + .any(|known| known.slot.eq_ignore_ascii_case(&slot)) + { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("slot {relation}.{slot} of macro {name}() is mapped twice"), + )); + } + let shown: String = format!("{relation}.{slot}"); + let mapped_columns: Vec = if group { + group_columns(name, &shown, argument.text.trim())? + } else { + vec![single_column(name, &shown, argument.text.trim())?] + }; + mapped[index].push(SlotMapping { + slot, + columns: mapped_columns, + group, + }); + } + for (index, parameter) in definition.parameters.iter().enumerate() { + let ParameterKind::Relation { columns, groups } = ¶meter.kind else { + continue; + }; + let relation: &str = ¶meter.name; + let given: Vec = std::mem::take(&mut mapped[index]); + let mut ordered: Vec = Vec::with_capacity(given.len()); + for (column, _) in columns { + if let Some(found) = mapping_of(&given, column) { + ordered.push(found.clone()); + } + } + for group in groups { + let Some(found) = mapping_of(&given, group).cloned() else { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("call to macro {name}() is missing slot {relation}.{group} COLUMNS"), + ) + .helped(format!( + "map it to the passed relation's columns, such as {relation}.{group} := (product_id)" + ))); + }; + ordered.push(found); + } + check_overlaps(name, relation, columns, &ordered)?; + mapped[index] = ordered; + } + Ok(mapped) +} + +/// The mapping the call made for `slot`, if it made one. +fn mapping_of<'m>(mappings: &'m [SlotMapping], slot: &str) -> Option<&'m SlotMapping> { + mappings + .iter() + .find(|mapping| mapping.slot.eq_ignore_ascii_case(slot)) +} + +/// The one column name a single-column slot is mapped to. +fn single_column(name: &str, shown: &str, text: &str) -> Result { + match tokens(text).as_deref() { + Ok([column]) if matches!(column.kind, TokenKind::Word | TokenKind::Quoted) => { + Ok(column.text(text).to_owned()) + } + _ => Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("slot {shown} of macro {name}() takes one column name of the passed relation; got {text}"), + ) + .helped(format!("map it to a column, such as {shown} := order_total; a slot reads the column as it is, so compute expressions in the relation you pass"))), + } +} + +/// The column names a `COLUMNS` slot is mapped to: a parenthesised list, never empty. +fn group_columns(name: &str, shown: &str, text: &str) -> Result, SqlMacroError> { + let invalid = || { + SqlMacroError::new( + ARGUMENT_CODE, + format!("slot {shown} of macro {name}() takes a parenthesised list of column names of the passed relation; got {text}"), + ) + .helped(format!("map it to columns, such as {shown} := (product_id, region)")) + }; + let scanned: Vec = tokens(text).map_err(|_| invalid())?; + let wrapped: bool = scanned.first().is_some_and(|open| open.is(text, "(")) + && closing_token(text, &scanned, 0) == Some(scanned.len() - 1); + if !wrapped { + return Err(invalid()); + } + let inner: &[Token] = &scanned[1..scanned.len() - 1]; + if inner.is_empty() { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("slot {shown} of macro {name}() needs at least one column; got {text}"), + ) + .helped(format!( + "map it to columns, such as {shown} := (product_id)" + ))); + } + let items: Vec<&[Token]> = split_top_level(text, inner); + let commas: usize = inner.iter().filter(|token| token.is(text, ",")).count(); + if items.len() != commas + 1 { + return Err(invalid()); + } + let mut columns: Vec = Vec::with_capacity(items.len()); + for item in items { + let [column] = item else { + return Err(invalid()); + }; + if !matches!(column.kind, TokenKind::Word | TokenKind::Quoted) { + return Err(invalid()); + } + let column: String = column.text(text).to_owned(); + if columns.iter().any(|known| same_column(known, &column)) { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!("column {column} is listed twice in slot {shown} of macro {name}()"), + )); + } + columns.push(column); + } + Ok(columns) +} + +/// Rejects groups of one relation sharing a column, or a group holding a single slot's column. +fn check_overlaps( + name: &str, + relation: &str, + columns: &[(String, String)], + mappings: &[SlotMapping], +) -> Result<(), SqlMacroError> { + let groups: Vec<&SlotMapping> = mappings.iter().filter(|mapping| mapping.group).collect(); + for (position, group) in groups.iter().enumerate() { + for other in &groups[position + 1..] { + if let Some(column) = shared_column(group, other) { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "slots {relation}.{} and {relation}.{} of macro {name}() both group column {column}; groups of one relation may not overlap", + group.slot, other.slot + ), + ) + .helped("list each column in at most one group")); + } + } + } + for (slot, _) in columns { + let read: String = mappings + .iter() + .find(|mapping| !mapping.group && mapping.slot.eq_ignore_ascii_case(slot)) + .map_or_else(|| slot.clone(), |mapping| mapping.columns[0].clone()); + if let Some(group) = groups.iter().find(|group| groups_column(group, &read)) { + return Err(SqlMacroError::new( + ARGUMENT_CODE, + format!( + "column {read} is in slot {relation}.{} and also fills single-column slot {relation}.{slot} of macro {name}(); a grouped column may not fill a single-column slot", + group.slot + ), + ) + .helped(format!( + "remove {read} from {relation}.{}, or map {relation}.{slot} to another column", + group.slot + ))); + } + } + Ok(()) +} + +/// The first column `group` shares with `other`, if any. +fn shared_column<'g>(group: &'g SlotMapping, other: &SlotMapping) -> Option<&'g String> { + group + .columns + .iter() + .find(|column| groups_column(other, column)) +} + +/// Whether `group` lists `column`. +fn groups_column(group: &SlotMapping, column: &str) -> bool { + group.columns.iter().any(|known| same_column(known, column)) +} + +/// Whether two column names written by a caller name the same column. +pub(crate) fn same_column(left: &str, right: &str) -> bool { + unquoted(left).eq_ignore_ascii_case(&unquoted(right)) +} + +/// A relation's slots as messages list them. +pub(crate) fn slot_list(columns: &[(String, String)], groups: &[String]) -> String { + columns + .iter() + .map(|(column, data_type)| format!("{column} {data_type}")) + .chain(groups.iter().map(|group| format!("{group} COLUMNS"))) + .collect::>() + .join(", ") +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs index 583231d189..e6463721e6 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/probes.rs @@ -10,10 +10,10 @@ use crate::sql_macros::_helpers::rendering::arguments::{fits, type_family}; use crate::sql_macros::_helpers::rendering::placement::{QueryParts, top_level_parts}; use crate::sql_macros::constants::{ ALIAS_PRECEDING_KEYWORDS, FROM_KEYWORD, INFERRED_TEXT_TYPE, PROBE_COLUMN_PREFIX, - PROBE_TAIL_ENDS, STAR_PROJECTION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, + PROBE_TAIL_ENDS, RELATION_FUNCTION_PREFIX, STAR_PROJECTION, UNKNOWN_TYPE_HINT, VARCHAR_TYPE, }; use crate::sql_macros::models::{ - ArgumentCheck, ArgumentExpectation, CallScope, SqlMacroCheck, SqlMacroFinding, + ArgumentCheck, ArgumentExpectation, CallScope, SlotMapping, SqlMacroCheck, SqlMacroFinding, }; use crate::sql_macros::types::{ProbeColumns, RelationShapes}; @@ -132,6 +132,17 @@ fn relation_probe(sql: &str, parts: &QueryParts, scope: &CallScope, name: &str) (!tail.is_empty()).then(|| format!("{prefix}SELECT {name}.* {tail}")) } +/// The probe listing the columns of `source`, a relation a subquery in a macro's SQL reads. +fn capture_probe(sql: &str, parts: &QueryParts, scope: &CallScope, source: &str) -> Option { + if source.starts_with(RELATION_FUNCTION_PREFIX) { + return Some(format!( + "{}SELECT * FROM {source}", + with_prefix(sql, parts, parts.ctes.len()) + )); + } + relation_probe(sql, parts, scope, source) +} + /// Whether the `FROM` text `tail` defines `name` as an alias, as `t AS name` or `t name`. fn defines_alias(tail: &str, name: &str) -> bool { let Ok(scanned) = tokens(tail) else { @@ -181,6 +192,17 @@ pub(crate) fn probe_plan(sql: &str, checks: &[SqlMacroCheck]) -> ProbePlan { let mut scopes: Vec> = Vec::new(); for (position, check) in checks.iter().enumerate() { let mut evidence: Vec> = vec![None; check.arguments.len()]; + for (index, argument) in check.arguments.iter().enumerate() { + if let ArgumentExpectation::Capture { + source: Some(source), + .. + } = &argument.expected + && let Some(probe) = capture_probe(sql, &parts, &check.scope, source) + { + plan.probes.push(probe); + evidence[index] = Some(Evidence::Columns(plan.probes.len() - 1)); + } + } if !check.plain_select { plan.evidence.push(evidence); continue; @@ -193,6 +215,7 @@ pub(crate) fn probe_plan(sql: &str, checks: &[SqlMacroCheck]) -> ProbePlan { evidence[index] = Some(Evidence::Columns(plan.probes.len() - 1)); } } + ArgumentExpectation::Capture { .. } => {} ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let group: usize = match typed_probe_index(&scopes, &check.scope) { Some(group) => group, @@ -213,7 +236,6 @@ pub(crate) fn probe_plan(sql: &str, checks: &[SqlMacroCheck]) -> ProbePlan { )); probe.arguments.push((position, index)); } - ArgumentExpectation::ColumnOf { .. } => {} } } plan.evidence.push(evidence); @@ -290,15 +312,41 @@ pub(crate) fn check_findings( for (index, argument) in check.arguments.iter().enumerate() { let parameter: &str = &argument.parameter; match &argument.expected { - ArgumentExpectation::Relation { columns } => { + ArgumentExpectation::Capture { + slot, + columns, + source, + } => { + let (message, help) = capture_finding( + (parameter, slot), + columns, + source.as_deref(), + columns_of(index), + ); + if let Some(message) = message { + findings.push(finding(argument, message, Some(help))); + } + } + ArgumentExpectation::Relation { columns, slots } => { let Some(found) = columns_of(index) else { continue; }; + if let Some(slot_finding) = slot_finding(argument, (columns, slots), found, dialect) + { + findings.push(finding(argument, slot_finding.0, Some(slot_finding.1))); + continue; + } let listed: Vec = columns .iter() .map(|(column, data_type)| format!("{column} {data_type}")) .collect(); for (column, data_type) in columns { + if slots + .iter() + .any(|mapping| mapping.slot.eq_ignore_ascii_case(column)) + { + continue; + } let reason: Option = match column_type(found, column) { None => Some(format!("{} has no column {column}", argument.text)), Some(actual) if !compatible(actual.as_deref(), data_type, dialect) => { @@ -320,31 +368,6 @@ pub(crate) fn check_findings( } } } - ArgumentExpectation::ColumnOf { relation } => { - let Some(source) = check - .arguments - .iter() - .position(|other| other.parameter == *relation) - else { - continue; - }; - let Some(found) = columns_of(source) else { - continue; - }; - if column_type(found, &argument.text).is_none() { - findings.push(finding( - argument, - format!( - "argument {parameter} of macro {name}() expects COLUMN OF {relation}; {} has no column {}", - check.arguments[source].text, argument.text - ), - Some(format!( - "pass one of the columns of {}", - check.arguments[source].text - )), - )); - } - } ArgumentExpectation::Typed { .. } | ArgumentExpectation::Family { .. } => { let Some(Some(Evidence::Column(probe, column))) = evidence.get(index) else { continue; @@ -394,6 +417,99 @@ pub(crate) fn check_findings( findings } +/// Why a plain group read in a subquery over `source` may bind elsewhere, if it may, and the fix. +fn capture_finding( + (relation, slot): (&str, &str), + columns: &[String], + source: Option<&str>, + found: Option<&Vec<(String, Option)>>, +) -> (Option, String) { + let help: String = format!( + "qualify the read with the alias of the relation it reads, as alias.@{relation}.{slot}, so its columns cannot bind to the subquery's relations" + ); + let Some(source) = source else { + return ( + Some(format!( + "slot {relation}.{slot} is read plainly in a subquery over an unnamed relation, whose columns cannot be checked; write alias.@{relation}.{slot}" + )), + help, + ); + }; + let Some(found) = found else { + return ( + Some(format!( + "slot {relation}.{slot} is read plainly in a subquery over {source}, whose columns are not known here; write alias.@{relation}.{slot}" + )), + help, + ); + }; + let captured: Option<&String> = columns + .iter() + .find(|column| column_type(found, column).is_some()); + let message: Option = captured.map(|column| { + format!("slot {relation}.{slot} column {column} would bind to {source} in this subquery; write alias.@{relation}.{slot}") + }); + (message, help) +} + +/// The first mapped slot whose column the passed relation lacks or holds with another type. +fn slot_finding( + argument: &ArgumentCheck, + (columns, slots): (&[(String, String)], &[SlotMapping]), + found: &[(String, Option)], + dialect: &str, +) -> Option<(String, String)> { + let relation: &str = &argument.parameter; + let passed: &str = &argument.text; + for mapping in slots { + let slot: &str = &mapping.slot; + let help: String = format!("map {relation}.{slot} to columns of {passed}"); + if mapping.group { + if let Some(missing) = mapping + .columns + .iter() + .find(|column| column_type(found, column).is_none()) + { + return Some(( + format!( + "slot {relation}.{slot} groups column {missing}; {passed} has no column {missing}" + ), + help, + )); + } + continue; + } + let Some((_, data_type)) = columns + .iter() + .find(|(column, _)| column.eq_ignore_ascii_case(slot)) + else { + continue; + }; + let column: &str = &mapping.columns[0]; + match column_type(found, column) { + None => { + return Some(( + format!( + "slot {relation}.{slot} needs a {data_type} column; {passed} has no column {column}" + ), + help, + )); + } + Some(actual) if !compatible(actual.as_deref(), data_type, dialect) => { + return Some(( + format!( + "slot {relation}.{slot} needs a {data_type} column; {passed}.{column} is {}", + shown_type(actual.as_deref().unwrap_or_default()) + ), + help, + )); + } + Some(_) => {} + } + } + None +} + /// An inferred type as messages show it, in the SQL type names schema declarations use. fn shown_type(inferred: &str) -> &str { if inferred.eq_ignore_ascii_case(INFERRED_TEXT_TYPE) { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs index 0f8e0d56e2..6b062e821e 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/typing/returns_check.rs @@ -49,10 +49,9 @@ fn stand_in_type(kind: &ParameterKind, families: bool) -> Option<&str> { ParameterKind::Any { family: Some(family), } if families => Some(family.widest()), - ParameterKind::Any { .. } - | ParameterKind::Enum { .. } - | ParameterKind::Relation { .. } - | ParameterKind::ColumnOf { .. } => None, + ParameterKind::Any { .. } | ParameterKind::Enum { .. } | ParameterKind::Relation { .. } => { + None + } } } @@ -70,10 +69,7 @@ fn body_probe( let mut arguments: Vec = Vec::with_capacity(definition.parameters.len()); let mut columns: Vec = Vec::with_capacity(definition.parameters.len()); for (index, parameter) in definition.parameters.iter().enumerate() { - if matches!( - parameter.kind, - ParameterKind::Relation { .. } | ParameterKind::ColumnOf { .. } - ) { + if matches!(parameter.kind, ParameterKind::Relation { .. }) { return None; } let column: String = stand_in_column(index); @@ -90,9 +86,10 @@ fn body_probe( sql, shape: ArgumentShape::Expression, calls: Vec::new(), + slots: Vec::new(), }); } - let expansion = match expand_body(definition, body, &arguments) { + let expansion = match expand_body((definition, None), body, &arguments) { Ok(expansion) => expansion, Err(_) => return None, }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/constants.rs b/crates/sqlbuild-analysis/src/sql_macros/constants.rs index 6d7397f960..27b708fa0a 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/constants.rs @@ -37,7 +37,13 @@ pub(crate) const RETURNS_KEY: &str = "returns"; pub(crate) const CONST_KEYWORD: &str = "CONST"; pub(crate) const DEFAULT_KEYWORD: &str = "DEFAULT"; pub(crate) const RELATION_KEYWORD: &str = "RELATION"; -pub(crate) const COLUMN_KEYWORD: &str = "COLUMN"; +/// The type word of a `RELATION` slot that groups caller-chosen columns. +pub(crate) const COLUMNS_KEYWORD: &str = "COLUMNS"; +/// The prefix of the word a single-column slot read stands as while a body is parsed. +pub(crate) const SLOT_PLACEHOLDER_PREFIX: &str = "__sqlbuild_slot__"; +/// Where a `COLUMNS` slot may stand, as messages list them. +pub(crate) const GROUP_POSITIONS: &str = + "a select list, PARTITION BY, GROUP BY, ORDER BY or USING (...)"; pub(crate) const OF_KEYWORD: &str = "OF"; pub(crate) const TYPE_KEYWORD: &str = "TYPE"; pub(crate) const ENUM_KEYWORD: &str = "ENUM"; @@ -140,3 +146,5 @@ pub(crate) const BIGQUERY_NUMERIC_LIMITS: (u32, u32) = (29, 9); pub(crate) const BIGQUERY_BIGNUMERIC_LIMITS: (u32, u32) = (38, 38); /// BigQuery's spellings of `BIGNUMERIC`. pub(crate) const BIGQUERY_BIG_DECIMAL_NAMES: [&str; 2] = ["BIGNUMERIC", "BIGDECIMAL"]; +/// How a relation function such as `__ref("orders")` starts. +pub(crate) const RELATION_FUNCTION_PREFIX: &str = "__"; diff --git a/crates/sqlbuild-analysis/src/sql_macros/models.rs b/crates/sqlbuild-analysis/src/sql_macros/models.rs index 7906f7aad1..cec625d827 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/models.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/models.rs @@ -37,10 +37,11 @@ pub enum ParameterKind { Collection { set: bool, data_type: String }, /// A member of the named enum. Enum { name: String }, - /// A relation with at least these columns and types. - Relation { columns: Vec<(String, String)> }, - /// A column name of the named relation parameter. - ColumnOf { relation: String }, + /// A relation and its slots: single typed columns and `COLUMNS` groups the caller chooses. + Relation { + columns: Vec<(String, String)>, + groups: Vec, + }, } impl ParameterKind { @@ -57,16 +58,46 @@ impl ParameterKind { format!("{} {data_type}", if *set { "SET" } else { "LIST" }) } Self::Enum { name } => format!("ENUM {name}"), - Self::Relation { columns } => { + Self::Relation { columns, groups } => { let listed: Vec = columns .iter() .map(|(name, data_type)| format!("{name} {data_type}")) + .chain(groups.iter().map(|group| format!("{group} COLUMNS"))) .collect(); format!("RELATION ({})", listed.join(", ")) } - Self::ColumnOf { relation } => format!("COLUMN OF {relation}"), } } + + /// Whether this is a `RELATION` that declares a `COLUMNS` slot called `slot`. + pub fn declares_group(&self, slot: &str) -> bool { + let Self::Relation { groups, .. } = self else { + return false; + }; + groups.iter().any(|group| group.eq_ignore_ascii_case(slot)) + } + + /// Whether this is a `RELATION` that declares a slot called `slot`. + pub fn declares_slot(&self, slot: &str) -> bool { + let Self::Relation { columns, groups } = self else { + return false; + }; + groups.iter().any(|group| group.eq_ignore_ascii_case(slot)) + || columns + .iter() + .any(|(column, _)| column.eq_ignore_ascii_case(slot)) + } +} + +/// The caller's columns one slot of a `RELATION` argument reads, as the call mapped them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SlotMapping { + /// The slot's name as the macro declares it. + pub slot: String, + /// The caller's columns: exactly one for a single-column slot, one or more for a group. + pub columns: Vec, + /// Whether the slot is a `COLUMNS` group. + pub group: bool, } /// A type family an `ANY` parameter may be narrowed to. @@ -185,6 +216,8 @@ pub struct SqlMacroLocation { #[derive(Clone, Debug, PartialEq, Eq)] pub struct CallArgument { pub name: Option, + /// The slot a `relation.slot := ...` argument maps; `name` is then the relation. + pub slot: Option, pub text: String, /// Byte offset of the argument text in the call's argument list. pub offset: usize, @@ -200,6 +233,8 @@ pub struct BoundArgument { pub from_default: bool, /// Byte offset of the argument in the call's argument list, when the call passed it. pub offset: Option, + /// The slots a `RELATION` argument's call maps, in the order the macro declares them. + pub slots: Vec, } /// The shape a typed argument was written in, which decides how it is checked and rendered. @@ -231,6 +266,8 @@ pub struct RenderedArgument { pub shape: ArgumentShape, /// The expression macro calls `sql` holds. pub calls: Vec, + /// The slots a `RELATION` argument's call maps. + pub slots: Vec, } /// One expansion's SQL and the names it defines, for clash checks at the call site. @@ -244,6 +281,20 @@ pub struct BodyExpansion { pub calls: Vec, /// The SQL of the argument whose type an expression macro call returns, by `returns TYPE OF`. pub like: Option, + /// Each plain `COLUMNS` slot read in a subquery, and the relations that could capture it. + pub captures: Vec, +} + +/// A plain `COLUMNS` slot read inside a subquery, whose bare columns could bind to its relations. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SlotCapture { + /// The `RELATION` parameter and the group read. + pub relation: String, + pub slot: String, + /// The group's columns at the call. + pub columns: Vec, + /// Each relation of the enclosing subqueries, innermost first; `None` for an unnamed one. + pub sources: Vec>, } /// One expression macro call's expansion in a text: its byte range and the macro's `returns`. @@ -390,14 +441,21 @@ pub struct ArgumentCheck { /// What an argument must be. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ArgumentExpectation { - /// A relation with at least these columns, each of a compatible type. - Relation { columns: Vec<(String, String)> }, - /// A column of the relation passed as the named parameter. - ColumnOf { relation: String }, + /// A relation with these columns, each of a compatible type, and its mapped slots' columns. + Relation { + columns: Vec<(String, String)>, + slots: Vec, + }, /// An expression of a type compatible with this one. Typed { data_type: String }, /// An expression of a type in this family. Family { family: AnyFamily }, + /// A relation, `None` when unnamed, that must hold none of a plain group read's columns. + Capture { + slot: String, + columns: Vec, + source: Option, + }, } /// A call argument the caller's analysed SQL shows does not fit its parameter. diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index 31340ce436..1f2cf23da5 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -4,6 +4,8 @@ mod helpers; mod test_definitions; #[path = "tests/test_expansion.rs"] mod test_expansion; +#[path = "tests/test_relation_slots.rs"] +mod test_relation_slots; #[path = "tests/test_typed_calls.rs"] mod test_typed_calls; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs index fc7b05b3df..1ece5d76c8 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_definitions.rs @@ -13,19 +13,25 @@ fn given_valid_macro_files_when_parsing_then_each_parameter_and_the_body_are_rea let test_cases = [ DefinitionTestCase { description: "every parameter type", - source: "MACRO (\n description \"Every type\",\n arguments (\n amount BIGINT,\n places INT CONST DEFAULT 2,\n codes LIST VARCHAR,\n unique_codes SET VARCHAR,\n status ENUM order_status,\n src RELATION (id BIGINT, created_at TIMESTAMP),\n key COLUMN OF src,\n price DECIMAL(18, 2),\n ),\n);\n\nWITH picked AS (SELECT @key FROM @src)\n", + source: "MACRO (\n description \"Every type\",\n arguments (\n amount BIGINT,\n places INT CONST DEFAULT 2,\n codes LIST VARCHAR,\n unique_codes SET VARCHAR,\n status ENUM order_status,\n src RELATION (id BIGINT, created_at TIMESTAMP, keys COLUMNS),\n price DECIMAL(18, 2),\n ),\n);\n\nWITH picked AS (SELECT @src.keys FROM @src)\n", expected_labels: &[ "BIGINT", "INT CONST", "LIST VARCHAR", "SET VARCHAR", "ENUM order_status", - "RELATION (id BIGINT, created_at TIMESTAMP)", - "COLUMN OF src", + "RELATION (id BIGINT, created_at TIMESTAMP, keys COLUMNS)", "DECIMAL(18, 2)", ], expected_form: "CTE-list", - expected_body_position: (15, 1), + expected_body_position: (14, 1), + }, + DefinitionTestCase { + description: "COLUMNS slots in every list position, single slots read directly and through aliases", + source: "MACRO (\n description \"Weekly rates\",\n arguments (weekly RELATION (keys COLUMNS, carried COLUMNS, order_count BIGINT)),\n);\n\nWITH counted AS (\n SELECT s.@weekly.keys, s.@weekly.carried, s.@weekly.order_count AS n\n FROM @weekly AS s\n),\nranked AS (\n SELECT @weekly.keys, n, ROW_NUMBER() OVER (PARTITION BY @weekly.keys ORDER BY @weekly.keys) AS r\n FROM counted\n),\nsummed AS (\n SELECT DISTINCT @weekly.keys, SUM(n) AS total FROM ranked GROUP BY @weekly.keys, n ORDER BY n, @weekly.keys\n),\njoined AS (SELECT * FROM summed JOIN counted USING (@weekly.keys))\n", + expected_labels: &["RELATION (order_count BIGINT, keys COLUMNS, carried COLUMNS)"], + expected_form: "CTE-list", + expected_body_position: (6, 1), }, DefinitionTestCase { description: "list parameter in PIVOT and runtime parameter in a WHERE list", @@ -143,10 +149,70 @@ fn given_invalid_macro_files_when_parsing_then_each_is_rejected() { expected_message: "argument src is a RELATION and cannot have a DEFAULT", }, DefinitionErrorTestCase { - description: "column of a non-relation", - source: "MACRO (description \"x\", arguments (src BIGINT, key COLUMN OF src));\nSELECT key", + description: "a slot declared twice", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS, KEYS BIGINT)));\nWITH a AS (SELECT * FROM @src)", + expected_code: "P015", + expected_message: "RELATION argument src declares slot KEYS twice", + }, + DefinitionErrorTestCase { + description: "a type constraint on a COLUMNS slot", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS OF VARCHAR)));\nWITH a AS (SELECT * FROM @src)", + expected_code: "P015", + expected_message: "slot src.keys COLUMNS takes nothing after it; a COLUMNS slot is untyped and each passed column keeps its own type", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot in a WHERE clause", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nWITH a AS (SELECT * FROM @src WHERE @src.keys IS NOT NULL)", expected_code: "P015", - expected_message: "argument key of macro m() is COLUMN OF src, but src is not a RELATION argument", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand; a COLUMNS slot is pasted as its columns, comma-separated, so it stands only as a whole item of a select list, PARTITION BY, GROUP BY, ORDER BY or USING (...)", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot inside a function call", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nWITH a AS (SELECT COUNT(DISTINCT @src.keys) AS n FROM @src)", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot renamed with AS", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nWITH a AS (SELECT s.@src.keys AS k FROM @src AS s)", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as s.@src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot ordered descending", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nWITH a AS (SELECT * FROM @src ORDER BY @src.keys DESC)", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot in an expression", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nWITH a AS (SELECT @src.keys || 'x' AS k FROM @src)", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot as an expression macro body", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\n@src.keys", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "a COLUMNS slot in a CASE branch", + source: "MACRO (description \"x\", arguments (src RELATION (keys COLUMNS)));\nSELECT CASE WHEN TRUE THEN @src.keys END AS k", + expected_code: "P015", + expected_message: "COLUMNS slot src.keys is read as @src.keys in macro m() where a list item cannot stand", + }, + DefinitionErrorTestCase { + description: "an aliased read naming no slot", + source: "MACRO (description \"x\", arguments (src RELATION (order_count BIGINT)));\nWITH a AS (SELECT s.@src.refund_count AS r FROM @src AS s)", + expected_code: "P022", + expected_message: "s.@src.refund_count in the body of macro m() names no slot of RELATION argument src", + }, + DefinitionErrorTestCase { + description: "an aliased read naming no slot at all", + source: "MACRO (description \"x\", arguments (src RELATION (order_count BIGINT)));\nWITH a AS (SELECT s.@src AS r FROM @src AS s)", + expected_code: "P022", + expected_message: "s.@src in the body of macro m() reads RELATION argument src through an alias without naming a slot", }, DefinitionErrorTestCase { description: "constant default that is not a literal", diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index d1cc4bf0cc..fe0701e975 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -33,7 +33,7 @@ const IN_SUPPORTED_COUNTRY: (&str, &str) = ( ); const LATEST_PER_KEY: (&str, &str) = ( "latest_per_key", - "MACRO (\n description \"Latest row per key\",\n arguments (\n src RELATION (updated_at TIMESTAMP),\n key COLUMN OF src,\n ),\n);\n\nWITH latest_ranked AS (\n SELECT @src.*, ROW_NUMBER() OVER (PARTITION BY @key ORDER BY updated_at DESC) AS row_num FROM @src\n),\nlatest_rows AS (SELECT * EXCLUDE (row_num) FROM latest_ranked WHERE row_num = 1)\n", + "MACRO (\n description \"Latest row per key\",\n arguments (\n src RELATION (updated_at TIMESTAMP, keys COLUMNS),\n ),\n);\n\nWITH latest_ranked AS (\n SELECT @src.*, ROW_NUMBER() OVER (PARTITION BY @src.keys ORDER BY updated_at DESC) AS row_num FROM @src\n),\nlatest_rows AS (SELECT * EXCLUDE (row_num) FROM latest_ranked WHERE row_num = 1)\n", ); const LATEST_LIVE_ORDERS: (&str, &str) = ( "latest_live_orders", @@ -247,7 +247,7 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { description: "__cte renames qualified references to a renamed CTE", adapter: "", macros: &[LATEST_PER_KEY], - sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), @sql.latest_per_key(r, id, __cte := l) SELECT * FROM l", + sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), @sql.latest_per_key(r, src.keys := (id), __cte := l) SELECT * FROM l", expected_sql: "WITH r AS (SELECT 1 AS id, 2 AS updated_at), l__latest_ranked AS (\n SELECT r.*, ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) AS row_num FROM r\n),\nl AS (SELECT * EXCLUDE (row_num) FROM l__latest_ranked WHERE row_num = 1) SELECT * FROM l", }, ExpansionTestCase { @@ -314,10 +314,10 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { expected_sql: "SELECT o.country_code IN ('GB', 'FR') AS supported FROM orders AS o", }, ExpansionTestCase { - description: "column of a relation", + description: "a COLUMNS slot of a relation", adapter: "", macros: &[LATEST_PER_KEY], - sql: "WITH customers_raw AS (SELECT 1 AS customer_id), @sql.latest_per_key(customers_raw, key := customer_id) SELECT * FROM latest_rows", + sql: "WITH customers_raw AS (SELECT 1 AS customer_id), @sql.latest_per_key(customers_raw, src.keys := (customer_id)) SELECT * FROM latest_rows", expected_sql: "WITH customers_raw AS (SELECT 1 AS customer_id), latest_ranked AS (\n SELECT customers_raw.*, ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY updated_at DESC) AS row_num FROM customers_raw\n),\nlatest_rows AS (SELECT * EXCLUDE (row_num) FROM latest_ranked WHERE row_num = 1) SELECT * FROM latest_rows", }, ExpansionTestCase { diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_relation_slots.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_relation_slots.rs new file mode 100644 index 0000000000..b38f3bc2bb --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_relation_slots.rs @@ -0,0 +1,295 @@ +use crate::sql_macros::tests::helpers::expanded_for; +use crate::sql_macros::tests::test_types::{ExpansionErrorTestCase, ExpansionTestCase}; + +const WEEKLY_RATES: (&str, &str) = ( + "weekly_rates", + "MACRO (\n description \"Weekly order and refund rates per grain\",\n arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)),\n);\n\nWITH counted AS (\n SELECT\n s.@weekly.keys,\n s.@weekly.order_count AS orders,\n s.@weekly.refund_count AS refunds,\n ROW_NUMBER() OVER (PARTITION BY s.@weekly.keys ORDER BY s.@weekly.order_count DESC) AS week_rank\n FROM @weekly AS s\n),\nrated AS (\n SELECT counted.@weekly.keys, SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n FROM counted\n GROUP BY counted.@weekly.keys\n ORDER BY counted.@weekly.keys\n)\n", +); +const RATE_COLUMNS: (&str, &str) = ( + "rate_columns", + "MACRO (\n description \"Grain and refund rate columns\",\n arguments (weekly RELATION (keys COLUMNS, refund_rate DOUBLE)),\n);\n\nSELECT @weekly.keys, @weekly.refund_rate AS rate\n", +); +const ORDER_COUNTS: (&str, &str) = ( + "order_counts", + "MACRO (\n description \"Order counts\",\n arguments (weekly RELATION (order_count BIGINT)),\n);\n\nSELECT @weekly.order_count\n", +); +const UNNAMED_COUNTS: (&str, &str) = ( + "unnamed_counts", + "MACRO (\n description \"Order counts plus one\",\n arguments (weekly RELATION (order_count BIGINT)),\n);\n\nWITH bumped AS (SELECT @weekly.order_count + 1 FROM @weekly)\n", +); +const SCALED_COUNT: (&str, &str) = ( + "scaled_count", + "MACRO (\n description \"Scaled order count\",\n arguments (weekly RELATION (order_count BIGINT), factor BIGINT),\n returns BIGINT,\n);\n\n@weekly.order_count * @factor\n", +); +const CARRIED_TOTALS: (&str, &str) = ( + "carried_totals", + "MACRO (\n description \"Totals per grain, carrying extra columns\",\n arguments (weekly RELATION (keys COLUMNS, carried COLUMNS, order_count BIGINT)),\n);\n\nWITH totals AS (\n SELECT @weekly.keys, @weekly.carried, SUM(@weekly.order_count) AS total\n FROM @weekly\n GROUP BY @weekly.keys, @weekly.carried\n)\n", +); +const JOINED_RATES: (&str, &str) = ( + "joined_rates", + "MACRO (\n description \"Rates joined back on the grain\",\n arguments (weekly RELATION (keys COLUMNS), rates RELATION (refund_rate DOUBLE)),\n);\n\nWITH joined AS (SELECT * FROM @weekly JOIN @rates USING (@weekly.keys))\n", +); +const FORWARDED_RATES: (&str, &str) = ( + "forwarded_rates", + "MACRO (\n description \"Weekly rates through another macro\",\n arguments (src RELATION (dims COLUMNS, placed BIGINT, refunded BIGINT)),\n);\n\nWITH @sql.weekly_rates(weekly := @src, weekly.keys := (dims), weekly.order_count := placed, weekly.refund_count := refunded)\n", +); +const FORWARDED_SAME_NAME: (&str, &str) = ( + "forwarded_same_name", + "MACRO (\n description \"Weekly rates through another macro with the same relation name\",\n arguments (weekly RELATION (dims COLUMNS, order_count BIGINT, refund_count BIGINT)),\n);\n\nWITH @sql.weekly_rates(weekly := @weekly, weekly.keys := (dims))\n", +); +const RATE_TOTAL: (&str, &str) = ( + "rate_total", + "MACRO (\n description \"Refund rate of one row\",\n arguments (weekly RELATION (order_count BIGINT, refund_count BIGINT)),\n returns DOUBLE,\n);\n\n@weekly.refund_count / NULLIF(@weekly.order_count, 0)\n", +); + +#[test] +fn given_relation_slots_when_expanding_then_each_read_becomes_the_mapped_columns() { + let test_cases = [ + ExpansionTestCase { + description: "a COLUMNS slot at one grain matches the hand-written per-product SQL", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS orders_placed, 0 AS refund_count),\n@sql.weekly_rates(weekly := weekly_orders, weekly.keys := (product_id), weekly.order_count := orders_placed)\nSELECT * FROM rated", + expected_sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS orders_placed, 0 AS refund_count),\ncounted AS (\n SELECT\n s.product_id,\n s.orders_placed AS orders,\n s.refund_count AS refunds,\n ROW_NUMBER() OVER (PARTITION BY s.product_id ORDER BY s.orders_placed DESC) AS week_rank\n FROM weekly_orders AS s\n),\nrated AS (\n SELECT counted.product_id, SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n FROM counted\n GROUP BY counted.product_id\n ORDER BY counted.product_id\n)\nSELECT * FROM rated", + }, + ExpansionTestCase { + description: "a COLUMNS slot at two columns prefixes every column with the alias", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 'EU' AS region, 2 AS order_count, 0 AS refund_count),\n@sql.weekly_rates(weekly := weekly_orders, weekly.keys := (product_id, region), __cte := by_region)\nSELECT * FROM by_region", + expected_sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 'EU' AS region, 2 AS order_count, 0 AS refund_count),\nby_region__counted AS (\n SELECT\n s.product_id, s.region,\n s.order_count AS orders,\n s.refund_count AS refunds,\n ROW_NUMBER() OVER (PARTITION BY s.product_id, s.region ORDER BY s.order_count DESC) AS week_rank\n FROM weekly_orders AS s\n),\nby_region AS (\n SELECT by_region__counted.product_id, by_region__counted.region, SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n FROM by_region__counted\n GROUP BY by_region__counted.product_id, by_region__counted.region\n ORDER BY by_region__counted.product_id, by_region__counted.region\n)\nSELECT * FROM by_region", + }, + ExpansionTestCase { + description: "plain group reads of a reference relation stand bare, unmapped slot reads qualified", + adapter: "", + macros: &[CARRIED_TOTALS], + sql: "WITH @sql.carried_totals(__ref(\"weekly_orders\"), weekly.keys := (product_id), weekly.carried := (region, channel)) SELECT * FROM totals", + expected_sql: "WITH totals AS (\n SELECT product_id, region, channel, SUM(weekly.order_count) AS total\n FROM __ref(\"weekly_orders\") AS weekly\n GROUP BY product_id, region, channel\n) SELECT * FROM totals", + }, + ExpansionTestCase { + description: "a COLUMNS slot in USING", + adapter: "", + macros: &[JOINED_RATES], + sql: "WITH a AS (SELECT 1 AS product_id), b AS (SELECT 1 AS product_id, 0.5 AS refund_rate), @sql.joined_rates(a, b, weekly.keys := (product_id)) SELECT * FROM joined", + expected_sql: "WITH a AS (SELECT 1 AS product_id), b AS (SELECT 1 AS product_id, 0.5 AS refund_rate), joined AS (SELECT * FROM a JOIN b USING (product_id)) SELECT * FROM joined", + }, + ExpansionTestCase { + description: "a column-list macro inserts a group's columns with their own names", + adapter: "", + macros: &[RATE_COLUMNS], + sql: "SELECT w.week, @sql.rate_columns(w, weekly.keys := (product_id, region)) FROM weekly_rates AS w", + expected_sql: "SELECT w.week, product_id, region, w.refund_rate AS rate FROM weekly_rates AS w", + }, + ExpansionTestCase { + description: "an unmapped single-column slot read keeps its own name, qualified by the relation", + adapter: "", + macros: &[ORDER_COUNTS], + sql: "SELECT @sql.order_counts(w) FROM weekly_orders AS w", + expected_sql: "SELECT w.order_count FROM weekly_orders AS w", + }, + ExpansionTestCase { + description: "a plain read of a mapped single slot reads its mapped column through the relation", + adapter: "", + macros: &[( + "counted_regions", + "MACRO (description \"Order counts and regions\", arguments (weekly RELATION (order_count BIGINT)));\n\nSELECT @weekly.order_count AS orders, @weekly.region", + )], + sql: "SELECT @sql.counted_regions(w, weekly.order_count := orders_placed) FROM weekly_orders AS w", + expected_sql: "SELECT w.orders_placed AS orders, w.region FROM weekly_orders AS w", + }, + ExpansionTestCase { + description: "an expression macro reads mapped single-column slots", + adapter: "", + macros: &[RATE_TOTAL], + sql: "SELECT @sql.rate_total(w, weekly.order_count := orders_placed) AS refund_rate FROM weekly_orders AS w", + expected_sql: "SELECT w.refund_count / NULLIF(w.orders_placed, 0) AS refund_rate FROM weekly_orders AS w", + }, + ExpansionTestCase { + description: "a relation forwarded to another macro maps the inner slots through the outer ones", + adapter: "", + macros: &[WEEKLY_RATES, FORWARDED_RATES], + sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS placed_count, 0 AS refunded_count),\n@sql.forwarded_rates(weekly_orders, src.dims := (product_id), src.placed := placed_count, src.refunded := refunded_count)\nSELECT * FROM rated", + expected_sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS placed_count, 0 AS refunded_count),\ncounted AS (\n SELECT\n s.product_id,\n s.placed_count AS orders,\n s.refunded_count AS refunds,\n ROW_NUMBER() OVER (PARTITION BY s.product_id ORDER BY s.placed_count DESC) AS week_rank\n FROM weekly_orders AS s\n),\nrated AS (\n SELECT counted.product_id, SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n FROM counted\n GROUP BY counted.product_id\n ORDER BY counted.product_id\n)\nSELECT * FROM rated", + }, + ExpansionTestCase { + description: "a relation forwarded under its own name maps the inner slots", + adapter: "", + macros: &[WEEKLY_RATES, FORWARDED_SAME_NAME], + sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS order_count, 0 AS refund_count),\n@sql.forwarded_same_name(weekly_orders, weekly.dims := (product_id))\nSELECT * FROM rated", + expected_sql: "WITH weekly_orders AS (SELECT 1 AS product_id, 2 AS order_count, 0 AS refund_count),\ncounted AS (\n SELECT\n s.product_id,\n s.order_count AS orders,\n s.refund_count AS refunds,\n ROW_NUMBER() OVER (PARTITION BY s.product_id ORDER BY s.order_count DESC) AS week_rank\n FROM weekly_orders AS s\n),\nrated AS (\n SELECT counted.product_id, SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n FROM counted\n GROUP BY counted.product_id\n ORDER BY counted.product_id\n)\nSELECT * FROM rated", + }, + ]; + + for test_case in test_cases { + let expanded = expanded_for(test_case.adapter, test_case.macros, test_case.sql) + .unwrap_or_else(|error| panic!("{}: {}", test_case.description, error.message)); + + assert_eq!( + expanded, test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_wrong_slot_mappings_when_expanding_then_each_call_fails_with_its_message() { + let test_cases = [ + ExpansionErrorTestCase { + description: "an unmapped COLUMNS slot", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w) SELECT * FROM rated", + expected_code: "P017", + expected_message: "call to macro weekly_rates() is missing slot weekly.keys COLUMNS", + }, + ExpansionErrorTestCase { + description: "an empty COLUMNS slot", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := ()) SELECT * FROM rated", + expected_code: "P017", + expected_message: "slot weekly.keys of macro weekly_rates() needs at least one column; got ()", + }, + ExpansionErrorTestCase { + description: "a COLUMNS slot mapped without parentheses", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := product_id) SELECT * FROM rated", + expected_code: "P017", + expected_message: "slot weekly.keys of macro weekly_rates() takes a parenthesised list of column names of the passed relation; got product_id", + }, + ExpansionErrorTestCase { + description: "a COLUMNS slot mapped to an expression", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (UPPER(region))) SELECT * FROM rated", + expected_code: "P017", + expected_message: "slot weekly.keys of macro weekly_rates() takes a parenthesised list of column names of the passed relation; got (UPPER(region))", + }, + ExpansionErrorTestCase { + description: "a column listed twice in one group", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (product_id, PRODUCT_ID)) SELECT * FROM rated", + expected_code: "P017", + expected_message: "column PRODUCT_ID is listed twice in slot weekly.keys of macro weekly_rates()", + }, + ExpansionErrorTestCase { + description: "a single-column slot mapped to an expression", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (product_id), weekly.order_count := orders + 1) SELECT * FROM rated", + expected_code: "P017", + expected_message: "slot weekly.order_count of macro weekly_rates() takes one column name of the passed relation; got orders + 1", + }, + ExpansionErrorTestCase { + description: "a slot the relation does not declare", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (product_id), weekly.region := region) SELECT * FROM rated", + expected_code: "P017", + expected_message: "RELATION argument weekly of macro weekly_rates() has no slot region", + }, + ExpansionErrorTestCase { + description: "a slot of an argument the macro does not have", + adapter: "", + macros: &[RATE_TOTAL], + sql: "SELECT @sql.rate_total(w, weekly.order_count := orders_placed, places.order_count := orders) AS r FROM weekly_orders AS w", + expected_code: "P017", + expected_message: "macro rate_total() has no argument places", + }, + ExpansionErrorTestCase { + description: "a slot mapped twice", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (product_id), weekly.keys := (region)) SELECT * FROM rated", + expected_code: "P017", + expected_message: "slot weekly.keys of macro weekly_rates() is mapped twice", + }, + ExpansionErrorTestCase { + description: "overlapping groups of one relation", + adapter: "", + macros: &[CARRIED_TOTALS], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.carried_totals(w, weekly.keys := (product_id, region), weekly.carried := (region)) SELECT * FROM totals", + expected_code: "P017", + expected_message: "slots weekly.keys and weekly.carried of macro carried_totals() both group column region; groups of one relation may not overlap", + }, + ExpansionErrorTestCase { + description: "a grouped column that also fills a mapped single-column slot", + adapter: "", + macros: &[CARRIED_TOTALS], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.carried_totals(w, weekly.keys := (product_id), weekly.carried := (units), weekly.order_count := units) SELECT * FROM totals", + expected_code: "P017", + expected_message: "column units is in slot weekly.carried and also fills single-column slot weekly.order_count of macro carried_totals(); a grouped column may not fill a single-column slot", + }, + ExpansionErrorTestCase { + description: "a grouped column that an unmapped single-column slot reads by name", + adapter: "", + macros: &[CARRIED_TOTALS], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.carried_totals(w, weekly.keys := (product_id, order_count), weekly.carried := (region)) SELECT * FROM totals", + expected_code: "P017", + expected_message: "column order_count is in slot weekly.keys and also fills single-column slot weekly.order_count of macro carried_totals(); a grouped column may not fill a single-column slot", + }, + ExpansionErrorTestCase { + description: "a mapped single-column slot read in a select list without AS", + adapter: "", + macros: &[ORDER_COUNTS], + sql: "SELECT @sql.order_counts(w, weekly.order_count := orders_placed) FROM weekly_orders AS w", + expected_code: "P017", + expected_message: "slot weekly.order_count of macro order_counts() is mapped to orders_placed, but the macro reads it in a select list without AS, so the output column's name would depend on the mapping", + }, + ExpansionErrorTestCase { + description: "a mapped single-column slot inside an unaliased select item", + adapter: "", + macros: &[UNNAMED_COUNTS], + sql: "WITH w AS (SELECT 1 AS orders_placed), @sql.unnamed_counts(w, weekly.order_count := orders_placed) SELECT * FROM bumped", + expected_code: "P017", + expected_message: "slot weekly.order_count of macro unnamed_counts() is mapped to orders_placed, but the macro reads it in a select list without AS, so the output column's name would depend on the mapping", + }, + ExpansionErrorTestCase { + description: "a slot of an argument that is not a relation", + adapter: "", + macros: &[SCALED_COUNT], + sql: "SELECT @sql.scaled_count(w, 2, factor.order_count := orders) AS r FROM weekly_orders AS w", + expected_code: "P017", + expected_message: "argument factor of macro scaled_count() is BIGINT, not a RELATION; only a RELATION argument has slots to map", + }, + ExpansionErrorTestCase { + description: "a grouped column that clashes with a column the macro defines", + adapter: "", + macros: &[WEEKLY_RATES], + sql: "WITH w AS (SELECT 1 AS product_id), @sql.weekly_rates(w, weekly.keys := (product_id, refund_rate)) SELECT * FROM rated", + expected_code: "P018", + expected_message: "column refund_rate of slot weekly.keys clashes with column refund_rate that macro weekly_rates() defines", + }, + ExpansionErrorTestCase { + description: "a grouped column that clashes with a column of the caller's select list", + adapter: "", + macros: &[RATE_COLUMNS], + sql: "SELECT w.region, @sql.rate_columns(w, weekly.keys := (product_id, region)) FROM weekly_rates AS w", + expected_code: "P018", + expected_message: "column \"region\" from macro rate_columns() clashes with a column already defined in model:orders_clean", + }, + ExpansionErrorTestCase { + description: "a positional argument after a slot mapping", + adapter: "", + macros: &[JOINED_RATES], + sql: "WITH a AS (SELECT 1 AS product_id), @sql.joined_rates(a, weekly.keys := (product_id), b) SELECT * FROM joined", + expected_code: "P017", + expected_message: "positional argument after a named argument in a call to macro joined_rates()", + }, + ]; + + for test_case in test_cases { + let error = expanded_for(test_case.adapter, test_case.macros, test_case.sql) + .expect_err(test_case.description); + + assert_eq!( + (error.code, error.message.as_str()), + (test_case.expected_code, test_case.expected_message), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs b/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs index ad0f2c0f42..b48f27d098 100644 --- a/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs +++ b/crates/sqlbuild-rules/src/dsl_headers/tests/test_dsl_headers.rs @@ -93,9 +93,9 @@ fn given_sql_macro_headers_when_formatting_then_each_is_laid_out_canonically_and }, FormatTestCase { description: "a multi-line header puts one item per line and breaks a list too wide", - text: "MACRO (description \"Latest row per key\",\n arguments (src RELATION (updated_at TIMESTAMP), key COLUMN OF src));\n\nWITH x AS (SELECT 1)\n", + text: "MACRO (description \"Latest row per key\",\n arguments (src RELATION (updated_at TIMESTAMP, keys COLUMNS), places INT));\n\nWITH x AS (SELECT 1)\n", line_width: 50, - expected_text: "MACRO (\n description \"Latest row per key\",\n arguments (\n src RELATION (updated_at TIMESTAMP),\n key COLUMN OF src,\n ),\n);\n\nWITH x AS (SELECT 1)\n", + expected_text: "MACRO (\n description \"Latest row per key\",\n arguments (\n src RELATION (\n updated_at TIMESTAMP,\n keys COLUMNS,\n ),\n places INT,\n ),\n);\n\nWITH x AS (SELECT 1)\n", }, FormatTestCase { description: "a header with a comment is left as authored", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py index 06b81ef83a..98ee578ab7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py @@ -103,3 +103,25 @@ class SqlMacroWarmCompileTestCase: replaced_text: tuple[str, str] expected_cold_fragment: str expected_warm_fragment: str + + +@dataclass(frozen=True) +class SqlMacroEquivalenceTestCase: + """A project built with two models whose compiled SQL and rows must be the same.""" + + description: str + files: dict[str, str] + compared_paths: tuple[str, str] + expected_fragment: str + compared_queries: tuple[str, str] + expected_row_count: int + + +@dataclass(frozen=True) +class SqlMacroFormatPairTestCase: + """A macro body reading slots and the same body written by hand, formatted together.""" + + description: str + slot_text: str + hand_text: str + expected_check_returncode: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index b071b39384..380835df9e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -103,10 +103,10 @@ LATEST_PER_KEY: dict[str, str] = { "macros/sql/latest_per_key.sql": ( 'MACRO (\n description "Latest row per key",\n' - " arguments (\n src RELATION (updated_at TIMESTAMP),\n key COLUMN OF src,\n " + " arguments (\n src RELATION (updated_at TIMESTAMP, keys COLUMNS),\n " "),\n);\n\n" "WITH latest_ranked AS (\n" - " SELECT @src.*, ROW_NUMBER() OVER (PARTITION BY @key ORDER BY updated_at DESC) AS " + " SELECT @src.*, ROW_NUMBER() OVER (PARTITION BY @src.keys ORDER BY updated_at DESC) AS " "row_num\n" " FROM @src\n),\n" "latest_rows AS (SELECT * EXCLUDE (row_num) FROM latest_ranked WHERE row_num = 1)\n" @@ -312,3 +312,245 @@ def picked_orders(body: str) -> dict[str, str]: "@sql.pick(raw, __cte := a)\nSELECT * FROM a\n" ), } + + +WEEKLY_ORDERS: dict[str, str] = { + "seeds/weekly_orders.csv": ( + "week,product_id,region,orders_placed,refund_count\n" + "1,10,EU,4,1\n1,10,US,6,0\n1,20,EU,5,2\n2,20,US,8,2\n" + ), + "seeds/weekly_orders.yml": ( + "seeds:\n - name: weekly_orders\n description: Weekly orders per product and region.\n" + " columns:\n" + " - name: week\n type: BIGINT\n" + " - name: product_id\n type: BIGINT\n" + " - name: region\n type: VARCHAR\n" + " - name: orders_placed\n type: BIGINT\n" + " - name: refund_count\n type: BIGINT\n" + ), +} + + +def rates_macro(name: str, arguments: str, keys: tuple[str, str], orders: str) -> dict[str, str]: + """A CTE-list macro of weekly refund rates reading its grain as `keys` (aliased, plain).""" + + aliased, plain = keys + return { + f"macros/sql/{name}.sql": ( + f'MACRO (\n description "Weekly refund rates",\n arguments ({arguments}),\n);\n\n' + "WITH counted AS (\n" + " SELECT\n" + f" s.{aliased.format(alias='s')},\n" + f" s.{orders} AS orders,\n" + " s.refund_count AS refunds,\n" + f" ROW_NUMBER() OVER (PARTITION BY {plain} ORDER BY s.week DESC) AS week_rank\n" + " FROM @weekly AS s\n" + "),\n\n" + "rated AS (\n" + " SELECT\n" + f" {plain},\n" + " SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate,\n" + " MAX(week_rank) AS weeks\n" + " FROM counted\n" + f" GROUP BY {plain}\n" + "),\n\n" + "rates_out AS (\n" + f" SELECT {plain}, refund_rate, weeks\n" + " FROM rated\n" + ")\n" + ) + } + + +SLOT_MACRO: dict[str, str] = rates_macro( + "weekly_rates", + "weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)", + ("@weekly.keys", "@weekly.keys"), + "@weekly.order_count", +) +PER_PRODUCT_MACRO: dict[str, str] = rates_macro( + "weekly_rates_per_product", + "weekly RELATION (product_id BIGINT, orders_placed BIGINT, refund_count BIGINT)", + ("product_id", "product_id"), + "orders_placed", +) +PER_PRODUCT_REGION_MACRO: dict[str, str] = rates_macro( + "weekly_rates_per_product_region", + "weekly RELATION (product_id BIGINT, region VARCHAR, orders_placed BIGINT, " + "refund_count BIGINT)", + ("product_id, {alias}.region", "product_id, region"), + "orders_placed", +) + + +def rates_model(name: str, per_product: str, per_product_region: str) -> dict[str, str]: + """A model reading weekly refund rates at two grains through the two given macro calls.""" + + return { + f"models/marts/{name}.sql": ( + 'MODEL (description "Refund rates per product, and per product and region");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + f"{per_product}\n" + f"{per_product_region}\n" + "final AS (\n" + " SELECT product_id, 'ALL' AS region, refund_rate FROM by_product\n" + " UNION ALL\n" + " SELECT product_id, region, refund_rate FROM by_product_region\n" + ")\n" + "SELECT * FROM final\n" + ) + } + + +BY_SLOT: dict[str, str] = rates_model( + "rates_by_slot", + "@sql.weekly_rates(weekly := weekly, weekly.keys := (product_id), " + "weekly.order_count := orders_placed, __cte := by_product)", + "@sql.weekly_rates(weekly := weekly, weekly.keys := (product_id, region), " + "weekly.order_count := orders_placed, __cte := by_product_region)", +) +BY_HAND: dict[str, str] = rates_model( + "rates_by_hand", + "@sql.weekly_rates_per_product(weekly, __cte := by_product)", + "@sql.weekly_rates_per_product_region(weekly, __cte := by_product_region)", +) + + +def slot_model(call: str) -> dict[str, str]: + """A model calling the weekly rates macro once with `call`'s mappings.""" + + return { + "models/marts/rates.sql": ( + 'MODEL (description "Refund rates");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + f"@sql.weekly_rates(weekly, {call})\n" + "SELECT * FROM rates_out\n" + ) + } + + +ORDER_COLUMNS: dict[str, str] = { + "macros/sql/order_columns.sql": ( + 'MACRO (\n description "Grain and order count columns",\n' + " arguments (weekly RELATION (keys COLUMNS, order_count BIGINT)),\n);\n\n" + "SELECT @weekly.keys, @weekly.order_count AS orders\n" + ) +} +UNNAMED_COUNT: dict[str, str] = { + "macros/sql/order_columns.sql": ( + 'MACRO (\n description "Order count column",\n' + " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" + "SELECT @weekly.order_count\n" + ) +} + + +LINEAGE_MODEL: dict[str, str] = { + "models/marts/order_counts.sql": ( + 'MODEL (description "Order counts per product and region");\n\n' + "SELECT @sql.order_columns(w, weekly.keys := (product_id, region), " + "weekly.order_count := orders_placed)\n" + 'FROM __seed("weekly_orders") AS w\n' + ) +} + + +def rate_model(name: str, expression: str) -> dict[str, str]: + """A model whose one `refund_rate` column is `expression` and is declared `VARCHAR`.""" + + return { + f"models/marts/{name}.sql": ( + 'MODEL (\n description "Refund rates",\n' + " contract enforced,\n columns (refund_rate (type VARCHAR)),\n);\n\n" + f"SELECT {expression} AS refund_rate\n" + 'FROM __seed("weekly_orders") AS w\n' + ) + } + + +REFUND_RATE_MACRO: dict[str, str] = { + "macros/sql/refund_rate.sql": ( + 'MACRO (\n description "Refund rate of one row",\n' + " arguments (weekly RELATION (order_count BIGINT, refund_count BIGINT)),\n);\n\n" + "@weekly.refund_count / NULLIF(@weekly.order_count, 0)\n" + ) +} +UNFORMATTED_SLOT_READS: str = ( + 'MACRO (description "Rates", arguments (weekly RELATION (keys COLUMNS, ' + "order_count BIGINT)));\n\n" + "WITH rated AS (select s.@weekly.keys, sum(s.@weekly.order_count) as orders " + "from @weekly as s group by s.@weekly.keys)\n" +) +HAND_WRITTEN_READS: tuple[tuple[str, str], ...] = ( + ("@weekly.keys", "product_id"), + ("s.@weekly.order_count", "s.order_count"), +) + + +def hand_written(text: str) -> str: + """`text` with each slot read written as the hand-written column it stands for.""" + + for slot_read, column in HAND_WRITTEN_READS: + text = text.replace(slot_read, column) + return text + + +REFUNDS: dict[str, str] = { + "seeds/refunds.csv": "week,product_id,refund_count\n1,10,1\n2,20,2\n", + "seeds/refunds.yml": ( + "seeds:\n - name: refunds\n description: Refunds per week and product.\n" + " columns:\n" + " - name: week\n type: BIGINT\n" + " - name: product_id\n type: BIGINT\n" + " - name: refund_count\n type: BIGINT\n" + ), +} + + +def refunded_weeks(query: str) -> dict[str, str]: + """A macro whose `refunded` CTE is `query`, and a model calling it on orders and refunds.""" + + return { + "macros/sql/refunded_weeks.sql": ( + 'MACRO (\n description "Weeks with refunds",\n' + " arguments (\n" + " weekly RELATION (keys COLUMNS, week BIGINT),\n" + " refunds RELATION (week BIGINT),\n" + " ),\n" + ");\n\n" + f"WITH refunded AS ({query})\n" + ), + "models/marts/refunded.sql": ( + 'MODEL (description "Refunded weeks");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + 'refunds AS (SELECT * FROM __seed("refunds")),\n' + "@sql.refunded_weeks(weekly, refunds, weekly.keys := (product_id))\n" + "SELECT * FROM refunded\n" + ), + } + + +def macro_file(name: str, arguments: str, body: str) -> dict[str, str]: + """One SQL macro file called `name`, declaring `arguments`, whose body is `body`.""" + + return { + f"macros/sql/{name}.sql": ( + f'MACRO (\n description "Macro {name}",\n arguments ({arguments}),\n);\n\n{body}\n' + ) + } + + +def scoped_model(macros: dict[str, str], body: str) -> dict[str, str]: + """Weekly orders, refunds, `macros`, and a model whose query after its two inputs is `body`.""" + + return { + **WEEKLY_ORDERS, + **REFUNDS, + **macros, + "models/marts/scoped.sql": ( + 'MODEL (description "Scoped slot reads");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + 'refunds AS (SELECT * FROM __seed("refunds")),\n' + f"{body}\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py index 7ae41d98ff..620db772a1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_build.py @@ -165,13 +165,13 @@ expected_rows=[(1, "2026-01-03 00:00:00", True)], ), SqlMacroBuildTestCase( - description="column_of_argument_and_list_default", + description="columns_slot_and_list_default", files={ **LATEST_PER_KEY, **IN_SUPPORTED_COUNTRY, **orders_model( 'WITH orders_raw AS (SELECT * FROM __ref("stg_orders")),\n' - "@sql.latest_per_key(orders_raw, key := status),\n" + "@sql.latest_per_key(orders_raw, src.keys := (status)),\n" "final AS (SELECT status, order_id FROM latest_rows " "WHERE @sql.in_supported_country(country_code))\n" "SELECT * FROM final\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py index 0e4a440594..59dd804657 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_errors.py @@ -304,16 +304,16 @@ ), ), SqlMacroErrorTestCase( - description="expression_for_column_of", + description="expression_for_a_columns_slot", files={ **LATEST_PER_KEY, **orders_model( - "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\n@sql.latest_per_key(orders_raw, key := status || 'x')\nSELECT * FROM latest_rows\n" + "WITH orders_raw AS (SELECT * FROM __ref(\"stg_orders\")),\n@sql.latest_per_key(orders_raw, src.keys := (status || 'x'))\nSELECT * FROM latest_rows\n" ), }, expected_code="P017", expected_fragments=( - "argument key of macro latest_per_key() expects COLUMN OF src: a column name of src", + "slot src.keys of macro latest_per_key() takes a parenthesised list of column names of the passed relation; got (status || 'x')", ), ), SqlMacroErrorTestCase( @@ -697,16 +697,16 @@ ), ), SqlMacroErrorTestCase( - description="column_of_naming_a_missing_column", + description="columns_slot_naming_a_missing_column", files={ **LATEST_PER_KEY, **orders_model( - 'WITH orders_raw AS (SELECT * FROM __ref("stg_orders")),\n@sql.latest_per_key(orders_raw, key := email_typo)\nSELECT * FROM latest_rows\n' + 'WITH orders_raw AS (SELECT * FROM __ref("stg_orders")),\n@sql.latest_per_key(orders_raw, src.keys := (status, email_typo))\nSELECT * FROM latest_rows\n' ), }, expected_code="P017", expected_fragments=( - "argument key of macro latest_per_key() expects COLUMN OF src; orders_raw has no column email_typo", + "slot src.keys groups column email_typo; orders_raw has no column email_typo", "models/marts/orders.sql:4:1", "macros/sql/latest_per_key.sql:", ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py new file mode 100644 index 0000000000..aca782590c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_relation_slots.py @@ -0,0 +1,533 @@ +"""E2E coverage for RELATION slots: COLUMNS groups and slots mapped at the call.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCommandTestCase, + SqlMacroCompileTestCase, + SqlMacroEquivalenceTestCase, + SqlMacroErrorTestCase, + SqlMacroFormatPairTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + BASE_FILES, + BY_HAND, + BY_SLOT, + DATABASE, + LINEAGE_MODEL, + ORDER_COLUMNS, + PER_PRODUCT_MACRO, + PER_PRODUCT_REGION_MACRO, + REFUND_RATE_MACRO, + REFUNDS, + SLOT_MACRO, + UNFORMATTED_SLOT_READS, + UNNAMED_COUNT, + WEEKLY_ORDERS, + hand_written, + inspected_text, + output, + rate_model, + refunded_weeks, + run_project, + slot_model, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb + +_RATES_QUERY: str = "SELECT product_id, region, refund_rate FROM {table} ORDER BY ALL" +_RATE_TYPE_MISMATCH: str = ( + "error[K002]: column 'refund_rate' inferred as DOUBLE but declared type is VARCHAR" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroEquivalenceTestCase( + description="columns_slot_at_two_grains_matches_per_grain_macros", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **PER_PRODUCT_MACRO, + **PER_PRODUCT_REGION_MACRO, + **BY_SLOT, + **BY_HAND, + }, + compared_paths=( + "target/compiled/models/marts/rates_by_slot.sql", + "target/compiled/models/marts/rates_by_hand.sql", + ), + expected_fragment=( + "ROW_NUMBER() OVER (PARTITION BY product_id, region ORDER BY s.week DESC)" + ), + compared_queries=( + _RATES_QUERY.format(table="rates_by_slot"), + _RATES_QUERY.format(table="rates_by_hand"), + ), + expected_row_count=6, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_columns_slot_at_two_grains_when_building_then_it_matches_per_grain_macros( + test_case: SqlMacroEquivalenceTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project(tmp_path=tmp_path, files=test_case.files, command=("build",)) + + assert result.returncode == 0, output(result) + by_slot: str = inspected_text( + project_dir=project_dir, relative_paths=test_case.compared_paths[:1] + ) + by_hand: str = inspected_text( + project_dir=project_dir, relative_paths=test_case.compared_paths[1:] + ) + assert by_slot == by_hand + assert test_case.expected_fragment in by_slot + rows: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / DATABASE, sql=test_case.compared_queries[0] + ) + assert rows == query_duckdb(db_path=project_dir / DATABASE, sql=test_case.compared_queries[1]) + assert len(rows) == test_case.expected_row_count + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroErrorTestCase( + description="single_column_slot_mapped_to_a_missing_column", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model("weekly.keys := (product_id), weekly.order_count := placed"), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: slot weekly.order_count needs a BIGINT column; weekly has no column placed", + "--> models/marts/rates.sql:4:1", + "--> macros/sql/weekly_rates.sql:3:14", + "help: map weekly.order_count to columns of weekly", + ), + ), + SqlMacroErrorTestCase( + description="single_column_slot_mapped_to_an_incompatible_column", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model("weekly.keys := (product_id), weekly.order_count := region"), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: slot weekly.order_count needs a BIGINT column; weekly.region is VARCHAR", + ), + ), + SqlMacroErrorTestCase( + description="columns_slot_naming_a_missing_column", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model( + "weekly.keys := (product_id, channel), weekly.order_count := orders_placed" + ), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: slot weekly.keys groups column channel; weekly has no column channel", + ), + ), + SqlMacroErrorTestCase( + description="unmapped_single_column_slot_reads_its_own_name", + files={**WEEKLY_ORDERS, **SLOT_MACRO, **slot_model("weekly.keys := (product_id)")}, + expected_code="P017", + expected_fragments=( + "error[P017]: argument weekly of macro weekly_rates() needs column order_count BIGINT; weekly has no column order_count", + ), + ), + SqlMacroErrorTestCase( + description="unmapped_columns_slot", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model("weekly.order_count := orders_placed"), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: call to macro weekly_rates() is missing slot weekly.keys COLUMNS", + "help: map it to the passed relation's columns, such as weekly.keys := (product_id)", + ), + ), + SqlMacroErrorTestCase( + description="grouped_column_that_also_fills_a_single_column_slot", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model( + "weekly.keys := (product_id, orders_placed), " + "weekly.order_count := orders_placed" + ), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: column orders_placed is in slot weekly.keys and also fills single-column slot weekly.order_count of macro weekly_rates(); a grouped column may not fill a single-column slot", + ), + ), + SqlMacroErrorTestCase( + description="grouped_column_named_like_a_column_the_macro_defines", + files={ + **WEEKLY_ORDERS, + **SLOT_MACRO, + **slot_model( + "weekly.keys := (product_id, refunds), weekly.order_count := orders_placed" + ), + }, + expected_code="P018", + expected_fragments=( + "error[P018]: column refunds of slot weekly.keys clashes with column refunds that macro weekly_rates() defines", + ), + ), + SqlMacroErrorTestCase( + description="mapped_single_column_slot_read_without_as", + files={ + **WEEKLY_ORDERS, + **UNNAMED_COUNT, + "models/marts/rates.sql": ( + 'MODEL (description "Order counts");\n\n' + "SELECT @sql.order_columns(w, weekly.order_count := orders_placed)\n" + 'FROM __seed("weekly_orders") AS w\n' + ), + }, + expected_code="P017", + expected_fragments=( + "error[P017]: slot weekly.order_count of macro order_columns() is mapped to orders_placed, but the macro reads it in a select list without AS, so the output column's name would depend on the mapping", + "help: add AS to @weekly.order_count in macros/sql/order_columns.sql", + ), + ), + SqlMacroErrorTestCase( + description="columns_slot_outside_a_list", + files={ + **WEEKLY_ORDERS, + "macros/sql/weekly_rates.sql": ( + 'MACRO (description "Rates", arguments (weekly RELATION (keys COLUMNS)));\n\n' + "WITH rated AS (SELECT * FROM @weekly WHERE @weekly.keys IS NOT NULL)\n" + ), + **slot_model("weekly.keys := (product_id)"), + }, + expected_code="P015", + expected_fragments=( + "error[P015]: COLUMNS slot weekly.keys is read as @weekly.keys in macro weekly_rates() where a list item cannot stand", + "--> macros/sql/weekly_rates.sql:3:44", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_wrong_slot_mappings_when_compiling_then_the_error_names_the_slot( + test_case: SqlMacroErrorTestCase, tmp_path: Path +) -> None: + _, result = run_project(tmp_path=tmp_path, files=test_case.files, command=("compile",)) + printed: str = output(result) + + assert result.returncode == 1, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCommandTestCase( + description="a_mapped_slot_traces_to_the_mapped_column", + files={**WEEKLY_ORDERS, **ORDER_COLUMNS, **LINEAGE_MODEL}, + command=("lineage", "order_counts.orders", "--direction", "upstream"), + expected_returncode=0, + expected_fragments=("weekly_orders.orders_placed",), + ), + SqlMacroCommandTestCase( + description="a_grouped_column_traces_to_its_own_column", + files={**WEEKLY_ORDERS, **ORDER_COLUMNS, **LINEAGE_MODEL}, + command=("lineage", "order_counts.region", "--direction", "upstream"), + expected_returncode=0, + expected_fragments=("weekly_orders.region",), + ), + SqlMacroCommandTestCase( + description="a_rule_sees_through_a_slot_read", + files={ + **WEEKLY_ORDERS, + "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] + + '\n[rules]\nselect = ["SQBRSQL001"]\n', + "macros/sql/order_columns.sql": ( + 'MACRO (\n description "Order count column",\n' + " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" + "SELECT @weekly.order_count = NULL AS has_no_orders\n" + ), + "models/marts/order_counts.sql": ( + 'MODEL (description "Order counts");\n\n' + "SELECT @sql.order_columns(w, weekly.order_count := orders_placed)\n" + 'FROM __seed("weekly_orders") AS w\n' + ), + }, + command=("compile",), + expected_returncode=1, + expected_fragments=( + "error[SQBRSQL001]: Comparison with NULL is never true (in SQL generated by " + "@sql.order_columns(w, weekly.order_count := orders_placed))", + "--> models/marts/order_counts.sql:3:8", + ), + ), + SqlMacroCommandTestCase( + description="an_aliased_slot_read_named_after_its_slot_is_no_self_alias", + files={ + **WEEKLY_ORDERS, + "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] + + '\n[rules]\nselect = ["SQBRSQL016"]\n', + "macros/sql/order_columns.sql": ( + 'MACRO (\n description "Order count column",\n' + " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" + "WITH counted AS (SELECT s.@weekly.order_count AS order_count FROM @weekly AS s),\n" + "named AS (SELECT @weekly.order_count AS order_count FROM @weekly)\n" + ), + "models/marts/order_counts.sql": ( + 'MODEL (description "Order counts");\n\n' + 'WITH w AS (SELECT * FROM __seed("weekly_orders")),\n' + "@sql.order_columns(w, weekly.order_count := orders_placed)\n" + "SELECT counted.order_count FROM counted, named\n" + ), + }, + command=("compile",), + expected_returncode=0, + expected_fragments=("Project compile OK",), + ), + SqlMacroCommandTestCase( + description="the_same_alias_on_a_hand_written_column_is_a_self_alias", + files={ + **WEEKLY_ORDERS, + "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] + + '\n[rules]\nselect = ["SQBRSQL016"]\n', + "macros/sql/order_columns.sql": ( + 'MACRO (\n description "Order count column",\n' + " arguments (weekly RELATION (order_count BIGINT)),\n);\n\n" + "WITH counted AS (SELECT s.order_count AS order_count FROM @weekly AS s)\n" + ), + "models/marts/order_counts.sql": ( + 'MODEL (description "Order counts");\n\n' + 'WITH w AS (SELECT orders_placed AS order_count FROM __seed("weekly_orders")),\n' + "@sql.order_columns(w)\n" + "SELECT counted.order_count FROM counted\n" + ), + }, + command=("compile",), + expected_returncode=1, + expected_fragments=( + "SQBRSQL016]: Expression is aliased to its existing name", + "--> macros/sql/order_columns.sql:6:", + ), + ), + SqlMacroCommandTestCase( + description="a_plain_slot_read_ambiguous_in_a_join_fails_like_hand_written_sql", + files={ + **WEEKLY_ORDERS, + "macros/sql/joined_keys.sql": ( + 'MACRO (\n description "Keys of joined weeks",\n' + " arguments (weekly RELATION (keys COLUMNS), other RELATION (week BIGINT)),\n);\n\n" + "WITH joined AS (\n" + " SELECT @weekly.keys\n" + " FROM @weekly AS w\n" + " JOIN @other AS o ON w.week = o.week\n" + ")\n" + ), + "models/marts/joined.sql": ( + 'MODEL (description "Joined weeks");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + "@sql.joined_keys(weekly, weekly, weekly.keys := (product_id))\n" + "SELECT * FROM joined\n" + ), + }, + command=("compile",), + expected_returncode=1, + expected_fragments=( + "error[B003]: Ambiguous unqualified column 'product_id' found in 2 referenced tables", + "--> models/marts/joined.sql:4:1", + ), + ), + SqlMacroCommandTestCase( + description="the_same_bare_name_written_by_hand_in_a_join", + files={ + **WEEKLY_ORDERS, + "models/marts/joined.sql": ( + 'MODEL (description "Joined weeks");\n\n' + 'WITH weekly AS (SELECT * FROM __seed("weekly_orders")),\n' + "joined AS (\n" + " SELECT product_id\n" + " FROM weekly AS w\n" + " JOIN weekly AS o ON w.week = o.week\n" + ")\n" + "SELECT * FROM joined\n" + ), + }, + command=("compile",), + expected_returncode=1, + expected_fragments=( + "error[B003]: Ambiguous unqualified column 'product_id' found in 2 referenced tables", + "--> models/marts/joined.sql:5:10", + ), + ), + SqlMacroCommandTestCase( + description="a_plain_group_read_in_a_correlated_subquery_that_captures_it", + files={ + **WEEKLY_ORDERS, + **REFUNDS, + **refunded_weeks( + "SELECT s.@weekly.keys FROM @weekly AS s\n" + "WHERE EXISTS (SELECT @weekly.keys FROM @refunds AS r WHERE r.week = s.week)" + ), + }, + command=("compile",), + expected_returncode=1, + expected_fragments=( + "error[P017]: slot weekly.keys column product_id would bind to refunds in this " + "subquery; write alias.@weekly.keys", + "--> models/marts/refunded.sql:5:1", + ), + ), + SqlMacroCommandTestCase( + description="the_same_subquery_reading_the_group_through_its_alias", + files={ + **WEEKLY_ORDERS, + **REFUNDS, + **refunded_weeks( + "SELECT s.@weekly.keys FROM @weekly AS s\n" + "WHERE EXISTS (SELECT s.@weekly.keys FROM @refunds AS r WHERE r.week = s.week)" + ), + }, + command=("compile",), + expected_returncode=0, + expected_fragments=("Project compiled",), + ), + SqlMacroCommandTestCase( + description="a_plain_group_read_in_a_cte_chain", + files={ + **WEEKLY_ORDERS, + **REFUNDS, + **refunded_weeks( + "SELECT s.@weekly.keys FROM @weekly AS s),\n" + "grouped AS (SELECT @weekly.keys FROM refunded GROUP BY @weekly.keys" + ), + }, + command=("compile",), + expected_returncode=0, + expected_fragments=("Project compiled",), + ), + SqlMacroCommandTestCase( + description="a_plain_group_read_beside_an_uncorrelated_subquery_and_in_a_derived_table", + files={ + **WEEKLY_ORDERS, + **REFUNDS, + **refunded_weeks( + "SELECT t.@weekly.keys FROM (SELECT @weekly.keys, week FROM @weekly) AS t\n" + "WHERE t.week IN (SELECT r.week FROM @refunds AS r)" + ), + }, + command=("compile",), + expected_returncode=0, + expected_fragments=("Project compiled",), + ), + ], + ids=lambda case: case.description, +) +def test_given_slot_reads_when_inspecting_then_tools_see_the_mapped_columns( + test_case: SqlMacroCommandTestCase, tmp_path: Path +) -> None: + _, result = run_project(tmp_path=tmp_path, files=test_case.files, command=test_case.command) + printed: str = output(result) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="expression_macro_reading_mapped_slots", + files={ + **WEEKLY_ORDERS, + **REFUND_RATE_MACRO, + **rate_model("rates", "@sql.refund_rate(w, weekly.order_count := orders_placed)"), + }, + expected_returncode=1, + expected_fragments=( + _RATE_TYPE_MISMATCH, + "w.refund_count / NULLIF(w.orders_placed, 0) AS refund_rate", + ), + inspected_paths=("target/compiled/models/marts/rates.sql",), + ), + SqlMacroCompileTestCase( + description="the_same_expression_written_by_hand", + files={ + **WEEKLY_ORDERS, + **rate_model("rates", "w.refund_count / NULLIF(w.orders_placed, 0)"), + }, + expected_returncode=1, + expected_fragments=( + _RATE_TYPE_MISMATCH, + "w.refund_count / NULLIF(w.orders_placed, 0) AS refund_rate", + ), + inspected_paths=("target/compiled/models/marts/rates.sql",), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_expression_macro_reading_slots_when_compiling_then_it_types_like_hand_sql( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile",) + ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroFormatPairTestCase( + description="slot_reads_and_columns_in_one_cte", + slot_text=UNFORMATTED_SLOT_READS, + hand_text=hand_written(UNFORMATTED_SLOT_READS), + expected_check_returncode=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_slot_reads_when_formatting_then_they_lay_out_like_hand_written_columns( + test_case: SqlMacroFormatPairTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, + files={ + "macros/sql/slots.sql": test_case.slot_text, + "macros/sql/by_hand.sql": test_case.hand_text, + }, + command=("format",), + ) + checked: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "format", "--check"), project_dir=project_dir + ) + + assert result.returncode == 0, output(result) + formatted: str = (project_dir / "macros/sql/slots.sql").read_text(encoding="utf-8") + assert formatted != test_case.slot_text + assert hand_written(formatted) == (project_dir / "macros/sql/by_hand.sql").read_text( + encoding="utf-8" + ) + assert checked.returncode == test_case.expected_check_returncode, output(checked) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_slot_scopes.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_slot_scopes.py new file mode 100644 index 0000000000..ed5a70a7fc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_slot_scopes.py @@ -0,0 +1,331 @@ +"""E2E coverage for slot reads across scopes: forwarding, subqueries, LATERAL and clashes.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + SLOT_MACRO, + inspected_text, + macro_file, + output, + run_project, + scoped_model, +) + +_COMPILED: tuple[str, ...] = ("target/compiled/models/marts/scoped.sql",) +_CAPTURED: str = ( + "error[P017]: slot weekly.keys column product_id would bind to refunds in this subquery; " + "write alias.@weekly.keys" +) +_JOINED_KEYS: dict[str, str] = macro_file( + "joined_keys", + "r RELATION (keys COLUMNS, wk BIGINT), other RELATION (week BIGINT)", + "WITH joined AS (SELECT * FROM @r JOIN @other USING (@r.keys, @r.wk))", +) +_JOINED_WEEKS: dict[str, str] = macro_file( + "joined_weeks", + "r RELATION (keys COLUMNS, week BIGINT), other RELATION (week BIGINT)", + "WITH joined AS (SELECT * FROM @r JOIN @other USING (@r.keys, @r.week))", +) +_KEYED_COUNTS: dict[str, str] = macro_file( + "keyed_counts", + "r RELATION (keys COLUMNS)", + "WITH counted AS (SELECT s.@r.keys, COUNT(*) AS n FROM @r AS s GROUP BY s.@r.keys),\n" + "listed AS (SELECT @r.keys, n FROM counted)", +) +_BUSY_ARGUMENTS: str = "weekly RELATION (order_count BIGINT, week BIGINT)" +_BUSY_CALL: str = "@sql.busy_weeks(weekly, weekly.order_count := orders_placed)\nSELECT * FROM busy" +_SCOPED_ARGUMENTS: str = ( + "weekly RELATION (keys COLUMNS, week BIGINT), refunds RELATION (week BIGINT)" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="forwarded_using_read_mapped_to_a_column_that_is_no_outer_slot", + files=scoped_model( + { + **_JOINED_KEYS, + **macro_file( + "outer_keys", + "w RELATION (dims COLUMNS), o RELATION (week BIGINT)", + "WITH @sql.joined_keys(r := @w, other := @o, r.keys := (dims), " + "r.wk := week)", + ), + }, + "@sql.outer_keys(weekly, refunds, w.dims := (product_id))\nSELECT * FROM joined", + ), + expected_returncode=0, + expected_fragments=("USING (product_id, week)",), + unexpected_fragments=("weekly.week",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="forwarded_using_read_of_an_unmapped_slot", + files=scoped_model( + { + **_JOINED_WEEKS, + **macro_file( + "outer_weeks", + "w RELATION (dims COLUMNS), o RELATION (week BIGINT)", + "WITH @sql.joined_weeks(r := @w, other := @o, r.keys := (dims))", + ), + }, + "@sql.outer_weeks(weekly, refunds, w.dims := (product_id))\nSELECT * FROM joined", + ), + expected_returncode=0, + expected_fragments=("USING (product_id, week)",), + unexpected_fragments=("weekly.week",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="forwarded_group_mapped_to_real_columns_keeps_its_kind", + files=scoped_model( + { + **_KEYED_COUNTS, + **macro_file( + "outer_counts", + "w RELATION (week BIGINT)", + "WITH @sql.keyed_counts(r := @w, r.keys := (product_id, region))", + ), + }, + "@sql.outer_counts(weekly)\nSELECT * FROM listed", + ), + expected_returncode=0, + expected_fragments=( + "SELECT s.product_id, s.region, COUNT(*) AS n FROM weekly AS s " + "GROUP BY s.product_id, s.region", + "SELECT product_id, region, n FROM counted", + ), + unexpected_fragments=("weekly.product_id", "P022"), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="forwarded_group_mapped_to_a_single_column_outer_slot", + files=scoped_model( + { + **_KEYED_COUNTS, + **macro_file( + "outer_counts", + "w RELATION (week BIGINT)", + "WITH @sql.keyed_counts(r := @w, r.keys := (week))", + ), + }, + "@sql.outer_counts(weekly)\nSELECT * FROM listed", + ), + expected_returncode=1, + expected_fragments=( + "slot r.keys of macro keyed_counts() is a COLUMNS slot, but the call maps it to " + "w.week, a single-column slot of the forwarded relation w", + ), + ), + SqlMacroCompileTestCase( + description="plain_read_of_an_undeclared_column_in_using", + files=scoped_model( + macro_file( + "joined_on_week", + "r RELATION (keys COLUMNS), other RELATION (refund_count BIGINT)", + "WITH joined AS (SELECT @r.keys FROM @r JOIN @other USING (@r.week))", + ), + "@sql.joined_on_week(weekly, refunds, r.keys := (region))\nSELECT * FROM joined", + ), + expected_returncode=0, + expected_fragments=("USING (week)",), + unexpected_fragments=("weekly.week",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="forwarded_slots_mapped_to_outer_slot_names", + files=scoped_model( + { + **SLOT_MACRO, + **macro_file( + "outer_rates", + "w RELATION (dims COLUMNS, placed BIGINT)", + "WITH @sql.weekly_rates(weekly := @w, weekly.keys := (dims), " + "weekly.order_count := placed)", + ), + }, + "@sql.outer_rates(weekly, w.dims := (product_id), w.placed := orders_placed)\n" + "SELECT * FROM rates_out", + ), + expected_returncode=0, + expected_fragments=( + "s.orders_placed AS orders", + "PARTITION BY product_id ORDER BY s.week DESC", + "GROUP BY product_id", + ), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="mapped_slot_without_as_in_an_in_subquery", + files=scoped_model( + macro_file( + "busy_weeks", + _BUSY_ARGUMENTS, + "WITH busy AS (SELECT s.week AS week FROM @weekly AS s\n" + "WHERE s.@weekly.order_count IN (SELECT w.@weekly.order_count FROM @weekly AS w))", + ), + _BUSY_CALL, + ), + expected_returncode=0, + expected_fragments=("IN (SELECT w.orders_placed FROM weekly AS w)",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="mapped_slot_without_as_in_an_exists_subquery", + files=scoped_model( + macro_file( + "busy_weeks", + _BUSY_ARGUMENTS, + "WITH busy AS (SELECT s.week AS week FROM @weekly AS s\n" + "WHERE EXISTS (SELECT w.@weekly.order_count FROM @weekly AS w " + "WHERE w.week = s.week))", + ), + _BUSY_CALL, + ), + expected_returncode=0, + expected_fragments=("EXISTS (SELECT w.orders_placed FROM weekly AS w",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="mapped_slot_without_as_in_a_scalar_subquery", + files=scoped_model( + macro_file( + "busy_weeks", + _BUSY_ARGUMENTS, + "WITH busy AS (SELECT s.week AS week,\n" + "(SELECT MAX(w.@weekly.order_count) FROM @weekly AS w) AS most FROM @weekly AS s)", + ), + _BUSY_CALL, + ), + expected_returncode=0, + expected_fragments=("(SELECT MAX(w.orders_placed) FROM weekly AS w) AS most",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="plain_group_read_in_a_comma_lateral_subquery_that_captures_it", + files=scoped_model( + macro_file( + "lateral_counts", + _SCOPED_ARGUMENTS, + "WITH counted AS (SELECT s.@weekly.keys, l.n FROM @weekly AS s,\n" + "LATERAL (SELECT COUNT(*) AS n FROM @refunds AS r WHERE r.week = s.week " + "GROUP BY @weekly.keys) AS l)", + ), + "@sql.lateral_counts(weekly, refunds, weekly.keys := (product_id))\n" + "SELECT * FROM counted", + ), + expected_returncode=1, + expected_fragments=(_CAPTURED, "--> models/marts/scoped.sql:5:1"), + ), + SqlMacroCompileTestCase( + description="plain_group_read_in_a_join_lateral_subquery_that_captures_it", + files=scoped_model( + macro_file( + "lateral_counts", + _SCOPED_ARGUMENTS, + "WITH counted AS (SELECT s.@weekly.keys, l.n FROM @weekly AS s\n" + "JOIN LATERAL (SELECT COUNT(*) AS n FROM @refunds AS r WHERE r.week = s.week " + "GROUP BY @weekly.keys) AS l ON TRUE)", + ), + "@sql.lateral_counts(weekly, refunds, weekly.keys := (product_id))\n" + "SELECT * FROM counted", + ), + expected_returncode=1, + expected_fragments=(_CAPTURED, "--> models/marts/scoped.sql:5:1"), + ), + SqlMacroCompileTestCase( + description="plain_group_read_in_a_subquery_over_the_passed_relation", + files=scoped_model( + macro_file( + "max_weeks", + _SCOPED_ARGUMENTS, + "WITH latest AS (SELECT s.@weekly.keys FROM @weekly AS s\n" + "WHERE s.week IN (SELECT MAX(week) FROM @weekly GROUP BY @weekly.keys))", + ), + "@sql.max_weeks(weekly, refunds, weekly.keys := (product_id))\n" + "SELECT * FROM latest", + ), + expected_returncode=0, + expected_fragments=("IN (SELECT MAX(week) FROM weekly GROUP BY product_id)",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="plain_group_read_in_a_subquery_over_the_passed_and_another_relation", + files=scoped_model( + macro_file( + "max_weeks", + _SCOPED_ARGUMENTS, + "WITH latest AS (SELECT s.@weekly.keys FROM @weekly AS s\n" + "WHERE s.week IN (SELECT MAX(r.week) FROM @weekly AS v JOIN @refunds AS r " + "ON v.week = r.week GROUP BY @weekly.keys))", + ), + "@sql.max_weeks(weekly, refunds, weekly.keys := (product_id))\n" + "SELECT * FROM latest", + ), + expected_returncode=1, + expected_fragments=(_CAPTURED, "--> models/marts/scoped.sql:5:1"), + ), + SqlMacroCompileTestCase( + description="aliased_group_read_in_a_column_list_macro", + files=scoped_model( + macro_file( + "order_columns", + "weekly RELATION (keys COLUMNS, order_count BIGINT)", + "SELECT o.@weekly.keys, o.@weekly.order_count AS orders", + ), + "counted AS (\n" + " SELECT @sql.order_columns(weekly, weekly.keys := (product_id, region), " + "weekly.order_count := orders_placed)\n" + " FROM weekly AS o\n" + ")\n" + "SELECT * FROM counted", + ), + expected_returncode=0, + expected_fragments=("o.product_id, o.region, o.orders_placed AS orders",), + inspected_paths=_COMPILED, + ), + SqlMacroCompileTestCase( + description="grouped_column_named_like_a_bare_column_of_the_same_select", + files=scoped_model( + macro_file( + "listed_weeks", + "weekly RELATION (keys COLUMNS)", + "WITH listed AS (SELECT s.week, s.@weekly.keys FROM @weekly AS s)", + ), + "@sql.listed_weeks(weekly, weekly.keys := (week, product_id))\n" + "SELECT * FROM listed", + ), + expected_returncode=1, + expected_fragments=( + "error[P018]: column week of slot weekly.keys clashes with column week that " + "macro listed_weeks() defines", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_slot_reads_across_scopes_when_compiling_then_each_resolves_or_fails_clearly( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project( + tmp_path=tmp_path, files=test_case.files, command=("compile",) + ) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py index a10d18b03e..64ca9488ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_since.py @@ -51,6 +51,31 @@ _SQL_MACROS: dict[str, str] = build_since_sql_macro_files() _INNER_SQL_MACRO: str = "models/_macros/sql/round_cents.sql" _OUTER_SQL_MACRO: str = "models/_macros/sql/net_dollars.sql" +_SLOT_MACRO: str = "models/_macros/sql/status_totals.sql" +_SLOT_MODEL: str = "models/order_status_totals.sql" +_SLOT_FILES: dict[str, str] = { + _SLOT_MACRO: ( + "MACRO (\n description 'Order totals per grain.',\n" + " arguments (src RELATION (keys COLUMNS, amount_cents INTEGER)),\n);\n\n" + "WITH totals AS (\n" + " SELECT @src.keys, SUM(@src.amount_cents) AS total_cents\n" + " FROM @src\n" + " GROUP BY @src.keys\n" + ")\n" + ), + _SLOT_MODEL: ( + "MODEL (description 'Order totals per status.');\n\n" + 'WITH raw AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.status_totals(raw, src.keys := (status))\n" + "SELECT * FROM totals\n" + ), + "tests/unit/test_order_status_totals.sql": ( + "TEST ();\n\nWITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id, 'open' AS status, 250 AS amount_cents),\n" + "__expected__order_status_totals AS (SELECT 'open' AS status, 250 AS total_cents)\n" + "SELECT 1\n" + ), +} @pytest.mark.parametrize( @@ -859,3 +884,74 @@ def test_given_sql_macro_changes_since_ref_when_testing_then_selects_tests_of_th assert fragment in output, output for fragment in test_case.unexpected_fragments: assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + SinceSelectionE2ETestCase( + description="a changed slot mapping changes the calling model", + edits={ + _SLOT_MODEL: _SLOT_FILES[_SLOT_MODEL].replace( + "src.keys := (status)", "src.keys := (status, order_id)" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 model changed", + "test_order_status_totals\n executes order_status_totals (model SQL changed)", + "PASS=1 FAIL=0 TOTAL=1", + ), + unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="a changed slot declaration changes the macro", + edits={ + _SLOT_MACRO: _SLOT_FILES[_SLOT_MACRO].replace( + "amount_cents INTEGER", "amount_cents BIGINT" + ) + }, + command=_EXPLAINED_TESTS, + expected_exit_code=0, + expected_fragments=( + "Changed since main: 1 macro changed", + "test_order_status_totals\n executes macro status_totals (changed)", + ), + unexpected_fragments=("test_stg_orders", "test_order_totals", "test_open_orders"), + ), + SinceSelectionE2ETestCase( + description="a reformatted slot mapping and slot read change nothing", + edits={ + _SLOT_MODEL: _SLOT_FILES[_SLOT_MODEL].replace( + "src.keys := (status)", "src.keys := ( status )" + ), + _SLOT_MACRO: _SLOT_FILES[_SLOT_MACRO].replace( + " GROUP BY @src.keys\n", " -- one row per grain\n GROUP BY @src.keys\n" + ), + }, + command=_TEST_COMMAND, + expected_exit_code=0, + expected_fragments=( + "Changed since main: no definition changes", + "No tests are affected by the changes since main", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_slot_changes_since_ref_when_testing_then_selects_tests_like_column_changes( + tmp_path: Path, test_case: SinceSelectionE2ETestCase +) -> None: + project: Path = prepare_since_repository(tmp_path=tmp_path, extra_files=_SLOT_FILES) + write_project_files(project_dir=project, files=test_case.edits) + run_git(repository=project, arguments=test_case.git_after_edits) + + result: CompletedProcess[str] = run_sqb(command=test_case.command, project_dir=project) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_fragments: + assert fragment in output, output + for fragment in test_case.unexpected_fragments: + assert fragment not in output, output From cce4cf298d259c7ae7c6faf81a6c887900f822de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:36:07 +0000 Subject: [PATCH 2/4] fix(rules): lint SQL macro slot reads without the slot's name as their column A mapped slot reads whatever column the call passes, so alias.@rel.slot AS slot is no self-alias. Aliased reads neutralize whole and bare reads hide the slot name. --- .../main/sql_macro_parameter_spans.rs | 35 +++++++++++++++---- .../src/lint_text/tests/test_lint_text.rs | 6 ++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs index 58a8a05f64..f216488829 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs @@ -1,16 +1,39 @@ //! The byte spans of a SQL macro body's `@name` parameter references, as expansion reads them. -use crate::sql_macros::_helpers::parsing::lexing::tokens; +use crate::sql_macros::_helpers::parsing::lexing::{Token, TokenKind, tokens}; use crate::sql_macros::_helpers::parsing::parameters::parameter_references; +use crate::sql_macros::_helpers::parsing::slots::prefix_start; -/// The `start..end` byte span of each `@name` in `body` that expansion replaces by an argument. +/// The span of each `@name` expansion replaces, and of each slot name whose column a call maps. #[must_use] pub fn sql_macro_parameter_spans(body: &str) -> Vec<(usize, usize)> { let Ok(scanned) = tokens(body) else { return Vec::new(); }; - parameter_references(body, &scanned) - .into_iter() - .map(|reference| (reference.start, reference.end)) - .collect() + let mut spans: Vec<(usize, usize)> = Vec::new(); + for reference in parameter_references(body, &scanned) { + let name: usize = reference.token; + let Some(slot) = slot_word(body, &scanned, name) else { + spans.push((reference.start, reference.end)); + continue; + }; + if name >= 1 && prefix_start(body, &scanned, name - 1).is_some() { + spans.push((reference.start, slot.end)); + continue; + } + spans.push((reference.start, reference.end)); + spans.push((slot.start, slot.end)); + } + spans +} + +/// The slot name directly after the `.` that follows the parameter name at token `name`, if any. +fn slot_word<'t>(body: &str, scanned: &'t [Token], name: usize) -> Option<&'t Token> { + let dot: &Token = scanned.get(name + 1)?; + let word: &Token = scanned.get(name + 2)?; + let joined: bool = dot.is(body, ".") + && dot.start == scanned[name].end + && matches!(word.kind, TokenKind::Word | TokenKind::Quoted) + && word.start == dot.end; + joined.then_some(word) } diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index ebadb2145c..001281c062 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -153,6 +153,12 @@ fn given_generated_parameters_when_neutralizing_then_only_bare_parameters_are_re neutralize: neutralize_sql_macro_parameters, expected_text: "CASE WHEN __sqlbuild_macro_parameter_0__ IN ('a', 'b') THEN 1 END + CASE WHEN __sqlbuild_macro_parameter_1__ IS NULL AND (__sqlbuild_macro_parameter_2__ IS NULL) THEN 1 END", }, + ParameterTestCase { + description: "SQL macro slot reads hide the slot name, whole after an alias", + body: "SELECT s.@src.amount AS amount, @src.id, @src.* FROM @src AS s", + neutralize: neutralize_sql_macro_parameters, + expected_text: "SELECT s.__sqlbuild_macro_parameter_0__ AS amount, __sqlbuild_macro_parameter_1__.__sqlbuild_macro_parameter_2__, __sqlbuild_macro_parameter_3__.* FROM __sqlbuild_macro_parameter_4__ AS s", + }, ParameterTestCase { description: "SQL macro calls, enum references and quoted text keep their at-signs", body: "@sql.other(@v) + @enum(\"s\").A + LENGTH('@v') + @@x", From dad7ba02ad47d75345d41800cc4c18a8e9a56f7a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:36:07 +0000 Subject: [PATCH 3/4] chore(benchmarks): move the duckshop benchmarks from COLUMN OF to COLUMNS slots --- benchmarks/duckshop-10k/checksum.txt | 2 +- benchmarks/duckshop-10k/duckshop10k/__init__.py | 2 +- .../duckshop-10k/duckshop10k/declarations.py | 14 ++++---------- benchmarks/duckshop-10k/duckshop10k/rules_gen.py | 7 ++++--- benchmarks/duckshop-10k/duckshop10k/staging.py | 4 ++-- .../staging/_sqlbuild/macros/sql/cdc_latest.sql | 7 ++----- .../models/staging/catalogue/stg_brands.sql | 2 +- .../staging/catalogue/stg_bundle_components.sql | 2 +- .../models/staging/catalogue/stg_bundles.sql | 2 +- .../models/staging/catalogue/stg_categories.sql | 2 +- .../models/staging/catalogue/stg_price_lists.sql | 2 +- .../models/staging/catalogue/stg_product_media.sql | 2 +- .../staging/catalogue/stg_product_variants.sql | 2 +- .../models/staging/catalogue/stg_products.sql | 2 +- .../staging/catalogue/stg_supplier_costs.sql | 2 +- .../models/staging/catalogue/stg_suppliers.sql | 2 +- .../models/staging/finance/stg_chargebacks.sql | 2 +- .../models/staging/finance/stg_invoices.sql | 2 +- .../models/staging/finance/stg_ledger_entries.sql | 2 +- .../project/models/staging/finance/stg_payouts.sql | 2 +- .../models/staging/finance/stg_tax_rates.sql | 2 +- .../staging/fulfilment/stg_carrier_rate_cards.sql | 2 +- .../staging/fulfilment/stg_delivery_exceptions.sql | 2 +- .../staging/fulfilment/stg_delivery_slots.sql | 2 +- .../models/staging/fulfilment/stg_pick_tasks.sql | 2 +- .../staging/fulfilment/stg_return_inspections.sql | 2 +- .../models/staging/fulfilment/stg_returns.sql | 2 +- .../staging/fulfilment/stg_shipment_packages.sql | 2 +- .../models/staging/fulfilment/stg_shipments.sql | 2 +- .../staging/growth/stg_affiliate_conversions.sql | 2 +- .../models/staging/growth/stg_campaign_spend.sql | 2 +- .../models/staging/growth/stg_campaigns.sql | 2 +- .../models/staging/growth/stg_influencer_posts.sql | 2 +- .../growth/stg_newsletter_subscriptions.sql | 2 +- .../models/staging/growth/stg_referral_invites.sql | 2 +- .../models/staging/growth/stg_survey_responses.sql | 2 +- .../models/staging/inventory/stg_bin_locations.sql | 2 +- .../models/staging/inventory/stg_po_receipts.sql | 2 +- .../staging/inventory/stg_purchase_orders.sql | 2 +- .../models/staging/inventory/stg_stock_counts.sql | 2 +- .../staging/inventory/stg_stock_reservations.sql | 2 +- .../staging/inventory/stg_stock_transfers.sql | 2 +- .../models/staging/inventory/stg_warehouses.sql | 2 +- .../staging/partners/stg_partner_listings.sql | 2 +- .../staging/partners/stg_partner_settlements.sql | 2 +- .../models/staging/storefront/stg_carts.sql | 2 +- .../staging/storefront/stg_customer_addresses.sql | 2 +- .../staging/storefront/stg_customer_consents.sql | 2 +- .../models/staging/storefront/stg_customers.sql | 2 +- .../storefront/stg_gift_card_transactions.sql | 2 +- .../models/staging/storefront/stg_gift_cards.sql | 2 +- .../staging/storefront/stg_loyalty_accounts.sql | 2 +- .../staging/storefront/stg_loyalty_ledger.sql | 2 +- .../models/staging/storefront/stg_order_lines.sql | 2 +- .../staging/storefront/stg_order_status_events.sql | 2 +- .../models/staging/storefront/stg_orders.sql | 2 +- .../models/staging/storefront/stg_payments.sql | 2 +- .../storefront/stg_promotion_redemptions.sql | 2 +- .../models/staging/storefront/stg_promotions.sql | 2 +- .../models/staging/storefront/stg_refunds.sql | 2 +- .../models/staging/storefront/stg_reviews.sql | 2 +- .../staging/storefront/stg_store_credit_ledger.sql | 2 +- .../staging/storefront/stg_subscription_events.sql | 2 +- .../staging/storefront/stg_subscriptions.sql | 2 +- .../models/staging/storefront/stg_wishlists.sql | 2 +- .../models/staging/support/stg_agent_shifts.sql | 2 +- .../project/models/staging/support/stg_agents.sql | 2 +- .../models/staging/support/stg_call_records.sql | 2 +- .../staging/support/stg_chat_transcripts.sql | 2 +- .../models/staging/support/stg_csat_responses.sql | 2 +- .../models/staging/support/stg_sla_policies.sql | 2 +- .../models/staging/support/stg_ticket_comments.sql | 2 +- .../project/models/staging/support/stg_tickets.sql | 2 +- .../duckshop/project/rules/staging_dedupe.py | 6 +++--- 74 files changed, 84 insertions(+), 92 deletions(-) diff --git a/benchmarks/duckshop-10k/checksum.txt b/benchmarks/duckshop-10k/checksum.txt index 3a8ae1527c..fb97b67c11 100644 --- a/benchmarks/duckshop-10k/checksum.txt +++ b/benchmarks/duckshop-10k/checksum.txt @@ -1 +1 @@ -generator 1 files 10874 sha256 4d276ad2428841efa2a1b851c83863bea70505d9e085a71390401399b5667667 +generator 2 files 10874 sha256 7dd651a573bce90f9816889cdeece75b9f837137813a033125daafa28b7b70eb diff --git a/benchmarks/duckshop-10k/duckshop10k/__init__.py b/benchmarks/duckshop-10k/duckshop10k/__init__.py index 4a2841094a..0a8894d664 100644 --- a/benchmarks/duckshop-10k/duckshop10k/__init__.py +++ b/benchmarks/duckshop-10k/duckshop10k/__init__.py @@ -4,6 +4,6 @@ regenerate checksum.txt, whenever a change alters any generated file. """ -GENERATOR_VERSION = "1" +GENERATOR_VERSION = "2" SEED = 20261024 PROJECT_NAME = "duckshop_10k" diff --git a/benchmarks/duckshop-10k/duckshop10k/declarations.py b/benchmarks/duckshop-10k/duckshop10k/declarations.py index 5045c3c252..5967a76ec2 100644 --- a/benchmarks/duckshop-10k/duckshop10k/declarations.py +++ b/benchmarks/duckshop-10k/duckshop10k/declarations.py @@ -8,16 +8,13 @@ SQL_MACROS: dict[str, str] = { "cdc_latest": """MACRO ( description "Latest surviving version of each key in an append-only change log", - arguments ( - src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ), - key COLUMN OF src, - ), + arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)), ); WITH cdc_versions AS ( SELECT * FROM @src - QUALIFY ROW_NUMBER() OVER (PARTITION BY @key ORDER BY _seq DESC) = 1 + QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key ORDER BY _seq DESC) = 1 ), cdc_latest AS ( @@ -28,16 +25,13 @@ """, "cdc_history": """MACRO ( description "Every version of each key in a change log, keeping the last one of each load", - arguments ( - src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ), - key COLUMN OF src, - ), + arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)), ); WITH cdc_history AS ( SELECT * FROM @src - QUALIFY ROW_NUMBER() OVER (PARTITION BY @key, _loaded_at ORDER BY _seq DESC) = 1 + QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key, _loaded_at ORDER BY _seq DESC) = 1 ) """, "load_metadata": """MACRO ( diff --git a/benchmarks/duckshop-10k/duckshop10k/rules_gen.py b/benchmarks/duckshop-10k/duckshop10k/rules_gen.py index 147748c00a..9010a21c86 100644 --- a/benchmarks/duckshop-10k/duckshop10k/rules_gen.py +++ b/benchmarks/duckshop-10k/duckshop10k/rules_gen.py @@ -32,9 +32,10 @@ def _is_change_log(ctx: RuleContext, source: str) -> bool: code="XSQBRDUCK001", message="Staging models over a change-log source must deduplicate it with a change-log macro", remediation=( - "Read the source in an import CTE and follow it with @sql.cdc_latest(, ), or " - "@sql.cdc_history(, ) for a model that keeps every version, so deletes and " - "superseded versions are handled the same way everywhere." + "Read the source in an import CTE and follow it with " + "@sql.cdc_latest(, src.key := ()), or " + "@sql.cdc_history(, src.key := ()) for a model that keeps every version, so " + "deletes and superseded versions are handled the same way everywhere." ), ) def staging_uses_change_log_macro(*, model: Model, ctx: RuleContext) -> list[Finding]: diff --git a/benchmarks/duckshop-10k/duckshop10k/staging.py b/benchmarks/duckshop-10k/duckshop10k/staging.py index 418d80dc67..6c1b751c07 100644 --- a/benchmarks/duckshop-10k/duckshop10k/staging.py +++ b/benchmarks/duckshop-10k/duckshop10k/staging.py @@ -202,7 +202,7 @@ def base_model(project: Project, instance: Instance, archetype: Archetype, folde ) relation = "deduplicated" else: - ctes.append(f"@sql.cdc_latest(src, {key})") + ctes.append(f"@sql.cdc_latest(src, src.key := ({key}))") relation = "cdc_latest" items: list[str] = [] columns: list[Column] = [] @@ -480,7 +480,7 @@ def history_model(project: Project, instance: Instance, archetype: Archetype, fo ) ctes = [ import_cte("src", f'__source("{source}")'), - f"@sql.cdc_history(src, {key_physical})", + f"@sql.cdc_history(src, src.key := ({key_physical}))", cte("versions", f"SELECT\n{select_list(versions)}\nFROM cdc_history AS s"), cte("final", f"SELECT\n{select_list(items)}\nFROM versions AS h"), ] diff --git a/benchmarks/duckshop/project/models/staging/_sqlbuild/macros/sql/cdc_latest.sql b/benchmarks/duckshop/project/models/staging/_sqlbuild/macros/sql/cdc_latest.sql index 119ee618f2..327b070552 100644 --- a/benchmarks/duckshop/project/models/staging/_sqlbuild/macros/sql/cdc_latest.sql +++ b/benchmarks/duckshop/project/models/staging/_sqlbuild/macros/sql/cdc_latest.sql @@ -1,15 +1,12 @@ MACRO ( description "Latest surviving version of each key in an append-only change log", - arguments ( - src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ), - key COLUMN OF src, - ), + arguments (src RELATION (_op VARCHAR, _seq NUMBER(38, 0), _loaded_at TIMESTAMP_NTZ, key COLUMNS)), ); WITH cdc_versions AS ( SELECT * FROM @src - QUALIFY ROW_NUMBER() OVER (PARTITION BY @key ORDER BY _seq DESC) = 1 + QUALIFY ROW_NUMBER() OVER (PARTITION BY @src.key ORDER BY _seq DESC) = 1 ), cdc_latest AS ( diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_brands.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_brands.sql index 3dd780f4de..efc35fe785 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_brands.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_brands.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("catalogue__brands") ), -@sql.cdc_latest(src, brand_code) +@sql.cdc_latest(src, src.key := (brand_code)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_bundle_components.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_bundle_components.sql index e3b9635229..f1f8c18f74 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_bundle_components.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_bundle_components.sql @@ -15,7 +15,7 @@ WITH src AS ( FROM __source("catalogue__bundle_components") ), -@sql.cdc_latest(src, component_id) +@sql.cdc_latest(src, src.key := (component_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_bundles.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_bundles.sql index cda8859373..ac38d4d89b 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_bundles.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_bundles.sql @@ -20,7 +20,7 @@ WITH src AS ( FROM __source("catalogue__bundles") ), -@sql.cdc_latest(src, bundle_sku) +@sql.cdc_latest(src, src.key := (bundle_sku)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_categories.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_categories.sql index d3efb76efc..1d7fc4574a 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_categories.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_categories.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("catalogue__categories") ), -@sql.cdc_latest(src, category_id) +@sql.cdc_latest(src, src.key := (category_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_price_lists.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_price_lists.sql index fef9332df9..d3d1a51cb7 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_price_lists.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_price_lists.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("catalogue__price_lists") ), -@sql.cdc_latest(src, price_list_id) +@sql.cdc_latest(src, src.key := (price_list_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_product_media.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_product_media.sql index 6f872fc406..9baae2ab46 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_product_media.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_product_media.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("catalogue__product_media") ), -@sql.cdc_latest(src, media_id) +@sql.cdc_latest(src, src.key := (media_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_product_variants.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_product_variants.sql index f808e0a8ff..2e40af4ed4 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_product_variants.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_product_variants.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("catalogue__product_variants") ), -@sql.cdc_latest(src, sku) +@sql.cdc_latest(src, src.key := (sku)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_products.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_products.sql index 3e6fa0e752..586565a72a 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_products.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_products.sql @@ -23,7 +23,7 @@ WITH src AS ( FROM __source("catalogue__products") ), -@sql.cdc_latest(src, product_id) +@sql.cdc_latest(src, src.key := (product_id)) paths AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_supplier_costs.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_supplier_costs.sql index f9ed7fa23a..ff00eb8a02 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_supplier_costs.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_supplier_costs.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("catalogue__supplier_costs") ), -@sql.cdc_latest(src, cost_id) +@sql.cdc_latest(src, src.key := (cost_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/catalogue/stg_suppliers.sql b/benchmarks/duckshop/project/models/staging/catalogue/stg_suppliers.sql index 704eba1121..65d9a8afd1 100644 --- a/benchmarks/duckshop/project/models/staging/catalogue/stg_suppliers.sql +++ b/benchmarks/duckshop/project/models/staging/catalogue/stg_suppliers.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("catalogue__suppliers") ), -@sql.cdc_latest(src, supplier_code) +@sql.cdc_latest(src, src.key := (supplier_code)) contacts AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/finance/stg_chargebacks.sql b/benchmarks/duckshop/project/models/staging/finance/stg_chargebacks.sql index 87bea6c1d7..263845e0f1 100644 --- a/benchmarks/duckshop/project/models/staging/finance/stg_chargebacks.sql +++ b/benchmarks/duckshop/project/models/staging/finance/stg_chargebacks.sql @@ -20,7 +20,7 @@ WITH src AS ( FROM __source("finance__chargebacks") ), -@sql.cdc_latest(src, chargeback_id) +@sql.cdc_latest(src, src.key := (chargeback_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/finance/stg_invoices.sql b/benchmarks/duckshop/project/models/staging/finance/stg_invoices.sql index fe3d87d2cf..24975c5b87 100644 --- a/benchmarks/duckshop/project/models/staging/finance/stg_invoices.sql +++ b/benchmarks/duckshop/project/models/staging/finance/stg_invoices.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("finance__invoices") ), -@sql.cdc_latest(src, invoice_id) +@sql.cdc_latest(src, src.key := (invoice_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/finance/stg_ledger_entries.sql b/benchmarks/duckshop/project/models/staging/finance/stg_ledger_entries.sql index 641942331d..f357783f1d 100644 --- a/benchmarks/duckshop/project/models/staging/finance/stg_ledger_entries.sql +++ b/benchmarks/duckshop/project/models/staging/finance/stg_ledger_entries.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("finance__ledger_entries") ), -@sql.cdc_latest(src, entry_id) +@sql.cdc_latest(src, src.key := (entry_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/finance/stg_payouts.sql b/benchmarks/duckshop/project/models/staging/finance/stg_payouts.sql index 3cdd4d60a3..f983424553 100644 --- a/benchmarks/duckshop/project/models/staging/finance/stg_payouts.sql +++ b/benchmarks/duckshop/project/models/staging/finance/stg_payouts.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("finance__payouts") ), -@sql.cdc_latest(src, payout_id) +@sql.cdc_latest(src, src.key := (payout_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/finance/stg_tax_rates.sql b/benchmarks/duckshop/project/models/staging/finance/stg_tax_rates.sql index aa871c8471..fd2d00445a 100644 --- a/benchmarks/duckshop/project/models/staging/finance/stg_tax_rates.sql +++ b/benchmarks/duckshop/project/models/staging/finance/stg_tax_rates.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("finance__tax_rates") ), -@sql.cdc_latest(src, tax_rate_id) +@sql.cdc_latest(src, src.key := (tax_rate_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_carrier_rate_cards.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_carrier_rate_cards.sql index 896227ce04..12c15bb8f6 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_carrier_rate_cards.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_carrier_rate_cards.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("fulfilment__carrier_rate_cards") ), -@sql.cdc_latest(src, rate_card_id) +@sql.cdc_latest(src, src.key := (rate_card_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_exceptions.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_exceptions.sql index 31772f67e2..f7435a67c3 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_exceptions.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_exceptions.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("fulfilment__delivery_exceptions") ), -@sql.cdc_latest(src, exception_id) +@sql.cdc_latest(src, src.key := (exception_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_slots.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_slots.sql index 35585c9249..0c79642360 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_slots.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_delivery_slots.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("fulfilment__delivery_slots") ), -@sql.cdc_latest(src, slot_id) +@sql.cdc_latest(src, src.key := (slot_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_pick_tasks.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_pick_tasks.sql index fc7348e450..d33223f952 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_pick_tasks.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_pick_tasks.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("fulfilment__pick_tasks") ), -@sql.cdc_latest(src, task_id) +@sql.cdc_latest(src, src.key := (task_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_return_inspections.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_return_inspections.sql index 7beaec7dd9..6f94049d36 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_return_inspections.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_return_inspections.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("fulfilment__return_inspections") ), -@sql.cdc_latest(src, inspection_id) +@sql.cdc_latest(src, src.key := (inspection_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_returns.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_returns.sql index 95de9fd473..b17a0fa5e4 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_returns.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_returns.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("fulfilment__returns") ), -@sql.cdc_latest(src, return_id) +@sql.cdc_latest(src, src.key := (return_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipment_packages.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipment_packages.sql index 3d5ab9996a..978566fb08 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipment_packages.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipment_packages.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("fulfilment__shipment_packages") ), -@sql.cdc_latest(src, package_id) +@sql.cdc_latest(src, src.key := (package_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipments.sql b/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipments.sql index d6c87b91f1..4aef29c7d9 100644 --- a/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipments.sql +++ b/benchmarks/duckshop/project/models/staging/fulfilment/stg_shipments.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("fulfilment__shipments") ), -@sql.cdc_latest(src, shipment_id) +@sql.cdc_latest(src, src.key := (shipment_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_affiliate_conversions.sql b/benchmarks/duckshop/project/models/staging/growth/stg_affiliate_conversions.sql index 2037efe308..16f5ab8c2f 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_affiliate_conversions.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_affiliate_conversions.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("growth__affiliate_conversions") ), -@sql.cdc_latest(src, conversion_id) +@sql.cdc_latest(src, src.key := (conversion_id)) clicks AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_campaign_spend.sql b/benchmarks/duckshop/project/models/staging/growth/stg_campaign_spend.sql index 4c96861308..b76d279488 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_campaign_spend.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_campaign_spend.sql @@ -20,7 +20,7 @@ WITH src AS ( FROM __source("growth__campaign_spend") ), -@sql.cdc_latest(src, spend_id) +@sql.cdc_latest(src, src.key := (spend_id)) metrics AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_campaigns.sql b/benchmarks/duckshop/project/models/staging/growth/stg_campaigns.sql index a0574c7c38..206996c49a 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_campaigns.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_campaigns.sql @@ -20,7 +20,7 @@ WITH src AS ( FROM __source("growth__campaigns") ), -@sql.cdc_latest(src, campaign_id) +@sql.cdc_latest(src, src.key := (campaign_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_influencer_posts.sql b/benchmarks/duckshop/project/models/staging/growth/stg_influencer_posts.sql index 6e2ba596c5..27a2d85d95 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_influencer_posts.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_influencer_posts.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("growth__influencer_posts") ), -@sql.cdc_latest(src, post_id) +@sql.cdc_latest(src, src.key := (post_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_newsletter_subscriptions.sql b/benchmarks/duckshop/project/models/staging/growth/stg_newsletter_subscriptions.sql index b9cb6f0c4b..5cfdb3a32f 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_newsletter_subscriptions.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_newsletter_subscriptions.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("growth__newsletter_subscriptions") ), -@sql.cdc_latest(src, list_subscription_id) +@sql.cdc_latest(src, src.key := (list_subscription_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_referral_invites.sql b/benchmarks/duckshop/project/models/staging/growth/stg_referral_invites.sql index 42a547276d..aa8521e9b8 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_referral_invites.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_referral_invites.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("growth__referral_invites") ), -@sql.cdc_latest(src, invite_id) +@sql.cdc_latest(src, src.key := (invite_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/growth/stg_survey_responses.sql b/benchmarks/duckshop/project/models/staging/growth/stg_survey_responses.sql index ed2e65343d..ef126176bc 100644 --- a/benchmarks/duckshop/project/models/staging/growth/stg_survey_responses.sql +++ b/benchmarks/duckshop/project/models/staging/growth/stg_survey_responses.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("growth__survey_responses") ), -@sql.cdc_latest(src, response_id) +@sql.cdc_latest(src, src.key := (response_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_bin_locations.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_bin_locations.sql index ceeed1e59b..da12d72352 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_bin_locations.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_bin_locations.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("inventory__bin_locations") ), -@sql.cdc_latest(src, bin_id) +@sql.cdc_latest(src, src.key := (bin_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_po_receipts.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_po_receipts.sql index c3dc9aeeb4..4d07456511 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_po_receipts.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_po_receipts.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("inventory__po_receipts") ), -@sql.cdc_latest(src, receipt_id) +@sql.cdc_latest(src, src.key := (receipt_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_purchase_orders.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_purchase_orders.sql index 54f3dd5956..08167a4a8d 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_purchase_orders.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_purchase_orders.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("inventory__purchase_orders") ), -@sql.cdc_latest(src, po_id) +@sql.cdc_latest(src, src.key := (po_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_counts.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_counts.sql index cc93362aac..59e9a05aef 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_counts.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_counts.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("inventory__stock_counts") ), -@sql.cdc_latest(src, count_id) +@sql.cdc_latest(src, src.key := (count_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_reservations.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_reservations.sql index e4b357e25e..370a8a4a7a 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_reservations.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_reservations.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("inventory__stock_reservations") ), -@sql.cdc_latest(src, reservation_id) +@sql.cdc_latest(src, src.key := (reservation_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_transfers.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_transfers.sql index 1417f21dc8..f7bd3da083 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_stock_transfers.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_stock_transfers.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("inventory__stock_transfers") ), -@sql.cdc_latest(src, transfer_id) +@sql.cdc_latest(src, src.key := (transfer_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/inventory/stg_warehouses.sql b/benchmarks/duckshop/project/models/staging/inventory/stg_warehouses.sql index f1cb59c348..0d30725968 100644 --- a/benchmarks/duckshop/project/models/staging/inventory/stg_warehouses.sql +++ b/benchmarks/duckshop/project/models/staging/inventory/stg_warehouses.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("inventory__warehouses") ), -@sql.cdc_latest(src, warehouse_code) +@sql.cdc_latest(src, src.key := (warehouse_code)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/partners/stg_partner_listings.sql b/benchmarks/duckshop/project/models/staging/partners/stg_partner_listings.sql index 0c798e9359..48b28b93e9 100644 --- a/benchmarks/duckshop/project/models/staging/partners/stg_partner_listings.sql +++ b/benchmarks/duckshop/project/models/staging/partners/stg_partner_listings.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("partners__partner_listings") ), -@sql.cdc_latest(src, listing_id) +@sql.cdc_latest(src, src.key := (listing_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/partners/stg_partner_settlements.sql b/benchmarks/duckshop/project/models/staging/partners/stg_partner_settlements.sql index 629c4d74a1..ea7f12a96f 100644 --- a/benchmarks/duckshop/project/models/staging/partners/stg_partner_settlements.sql +++ b/benchmarks/duckshop/project/models/staging/partners/stg_partner_settlements.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("partners__partner_settlements") ), -@sql.cdc_latest(src, settlement_id) +@sql.cdc_latest(src, src.key := (settlement_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_carts.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_carts.sql index 22dfaefb08..8f865255df 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_carts.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_carts.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__carts") ), -@sql.cdc_latest(src, cart_id) +@sql.cdc_latest(src, src.key := (cart_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_customer_addresses.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_customer_addresses.sql index 5667965d64..d79ea68d34 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_customer_addresses.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_customer_addresses.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__customer_addresses") ), -@sql.cdc_latest(src, address_id) +@sql.cdc_latest(src, src.key := (address_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_customer_consents.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_customer_consents.sql index f94d060918..0bf45c23a2 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_customer_consents.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_customer_consents.sql @@ -26,7 +26,7 @@ customers AS ( FROM __source("storefront__customers") ), -@sql.cdc_latest(src, consent_id) +@sql.cdc_latest(src, src.key := (consent_id)) explicit_consents AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_customers.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_customers.sql index 9711dc4f6c..d89241c307 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_customers.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_customers.sql @@ -23,7 +23,7 @@ WITH src AS ( FROM __source("storefront__customers") ), -@sql.cdc_latest(src, customer_id) +@sql.cdc_latest(src, src.key := (customer_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_gift_card_transactions.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_gift_card_transactions.sql index c2d75f0c17..bd8b2cfa56 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_gift_card_transactions.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_gift_card_transactions.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("storefront__gift_card_transactions") ), -@sql.cdc_latest(src, transaction_id) +@sql.cdc_latest(src, src.key := (transaction_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_gift_cards.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_gift_cards.sql index 5884470032..a6a2372053 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_gift_cards.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_gift_cards.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("storefront__gift_cards") ), -@sql.cdc_latest(src, gift_card_id) +@sql.cdc_latest(src, src.key := (gift_card_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_accounts.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_accounts.sql index d410a2b147..564c897af4 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_accounts.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_accounts.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("storefront__loyalty_accounts") ), -@sql.cdc_latest(src, loyalty_account_id) +@sql.cdc_latest(src, src.key := (loyalty_account_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_ledger.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_ledger.sql index 0402441c25..92b26ac8c3 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_ledger.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_loyalty_ledger.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__loyalty_ledger") ), -@sql.cdc_latest(src, entry_id) +@sql.cdc_latest(src, src.key := (entry_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_order_lines.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_order_lines.sql index 462e2210e1..22f5188405 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_order_lines.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_order_lines.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__order_lines") ), -@sql.cdc_latest(src, order_line_id) +@sql.cdc_latest(src, src.key := (order_line_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_order_status_events.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_order_status_events.sql index 89111e6a93..1d71794ff3 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_order_status_events.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_order_status_events.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("storefront__order_status_events") ), -@sql.cdc_latest(src, event_id) +@sql.cdc_latest(src, src.key := (event_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_orders.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_orders.sql index d14172053e..b2354b1cc6 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_orders.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_orders.sql @@ -22,7 +22,7 @@ WITH src AS ( FROM __source("storefront__orders") ), -@sql.cdc_latest(src, order_id) +@sql.cdc_latest(src, src.key := (order_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_payments.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_payments.sql index 7372a5d73f..e144224ffa 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_payments.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_payments.sql @@ -24,7 +24,7 @@ WITH src AS ( FROM __source("storefront__payments") ), -@sql.cdc_latest(src, payment_id) +@sql.cdc_latest(src, src.key := (payment_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_promotion_redemptions.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_promotion_redemptions.sql index dd2d3f2c24..e689cfaa0b 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_promotion_redemptions.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_promotion_redemptions.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("storefront__promotion_redemptions") ), -@sql.cdc_latest(src, redemption_id) +@sql.cdc_latest(src, src.key := (redemption_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_promotions.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_promotions.sql index e67978d861..04f8a72a3f 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_promotions.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_promotions.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__promotions") ), -@sql.cdc_latest(src, promo_code) +@sql.cdc_latest(src, src.key := (promo_code)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_refunds.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_refunds.sql index 9da99645ca..9a86aca0da 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_refunds.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_refunds.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("storefront__refunds") ), -@sql.cdc_latest(src, refund_id) +@sql.cdc_latest(src, src.key := (refund_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_reviews.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_reviews.sql index c96f783726..05344ea392 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_reviews.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_reviews.sql @@ -19,7 +19,7 @@ WITH src AS ( FROM __source("storefront__reviews") ), -@sql.cdc_latest(src, review_id) +@sql.cdc_latest(src, src.key := (review_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_store_credit_ledger.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_store_credit_ledger.sql index 63fe21e56d..9c65e1823f 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_store_credit_ledger.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_store_credit_ledger.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("storefront__store_credit_ledger") ), -@sql.cdc_latest(src, entry_id) +@sql.cdc_latest(src, src.key := (entry_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_subscription_events.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_subscription_events.sql index 22cba6f84c..4dfc8edc44 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_subscription_events.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_subscription_events.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("storefront__subscription_events") ), -@sql.cdc_latest(src, event_id) +@sql.cdc_latest(src, src.key := (event_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_subscriptions.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_subscriptions.sql index d15fdf39db..2cc1fc5d4d 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_subscriptions.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_subscriptions.sql @@ -20,7 +20,7 @@ WITH src AS ( FROM __source("storefront__subscriptions") ), -@sql.cdc_latest(src, subscription_id) +@sql.cdc_latest(src, src.key := (subscription_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/storefront/stg_wishlists.sql b/benchmarks/duckshop/project/models/staging/storefront/stg_wishlists.sql index ffd454d6c2..73e0b3f005 100644 --- a/benchmarks/duckshop/project/models/staging/storefront/stg_wishlists.sql +++ b/benchmarks/duckshop/project/models/staging/storefront/stg_wishlists.sql @@ -14,7 +14,7 @@ WITH src AS ( FROM __source("storefront__wishlists") ), -@sql.cdc_latest(src, wishlist_id) +@sql.cdc_latest(src, src.key := (wishlist_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_agent_shifts.sql b/benchmarks/duckshop/project/models/staging/support/stg_agent_shifts.sql index cdf2f380c0..a0e32a1393 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_agent_shifts.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_agent_shifts.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("support__agent_shifts") ), -@sql.cdc_latest(src, shift_id) +@sql.cdc_latest(src, src.key := (shift_id)) breaks AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_agents.sql b/benchmarks/duckshop/project/models/staging/support/stg_agents.sql index 9bb7987cec..22f126aa37 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_agents.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_agents.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("support__agents") ), -@sql.cdc_latest(src, agent_id) +@sql.cdc_latest(src, src.key := (agent_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_call_records.sql b/benchmarks/duckshop/project/models/staging/support/stg_call_records.sql index d9a6bd10ae..ee351cafcf 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_call_records.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_call_records.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("support__call_records") ), -@sql.cdc_latest(src, call_id) +@sql.cdc_latest(src, src.key := (call_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_chat_transcripts.sql b/benchmarks/duckshop/project/models/staging/support/stg_chat_transcripts.sql index dc80b46b99..03a179fd27 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_chat_transcripts.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_chat_transcripts.sql @@ -15,7 +15,7 @@ WITH src AS ( FROM __source("support__chat_transcripts") ), -@sql.cdc_latest(src, chat_id) +@sql.cdc_latest(src, src.key := (chat_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_csat_responses.sql b/benchmarks/duckshop/project/models/staging/support/stg_csat_responses.sql index 559eb1eab3..a0a0e23a51 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_csat_responses.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_csat_responses.sql @@ -17,7 +17,7 @@ WITH src AS ( FROM __source("support__csat_responses") ), -@sql.cdc_latest(src, response_id) +@sql.cdc_latest(src, src.key := (response_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_sla_policies.sql b/benchmarks/duckshop/project/models/staging/support/stg_sla_policies.sql index 49fc28c2a4..02d3a1b342 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_sla_policies.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_sla_policies.sql @@ -16,7 +16,7 @@ WITH src AS ( FROM __source("support__sla_policies") ), -@sql.cdc_latest(src, policy_id) +@sql.cdc_latest(src, src.key := (policy_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_ticket_comments.sql b/benchmarks/duckshop/project/models/staging/support/stg_ticket_comments.sql index aeb842458b..b7ed303920 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_ticket_comments.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_ticket_comments.sql @@ -18,7 +18,7 @@ WITH src AS ( FROM __source("support__ticket_comments") ), -@sql.cdc_latest(src, comment_id) +@sql.cdc_latest(src, src.key := (comment_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/models/staging/support/stg_tickets.sql b/benchmarks/duckshop/project/models/staging/support/stg_tickets.sql index 31735c4237..530d9e5c49 100644 --- a/benchmarks/duckshop/project/models/staging/support/stg_tickets.sql +++ b/benchmarks/duckshop/project/models/staging/support/stg_tickets.sql @@ -25,7 +25,7 @@ WITH src AS ( FROM __source("support__tickets") ), -@sql.cdc_latest(src, ticket_id) +@sql.cdc_latest(src, src.key := (ticket_id)) final AS ( SELECT diff --git a/benchmarks/duckshop/project/rules/staging_dedupe.py b/benchmarks/duckshop/project/rules/staging_dedupe.py index 093fbf5413..1a41f278c2 100644 --- a/benchmarks/duckshop/project/rules/staging_dedupe.py +++ b/benchmarks/duckshop/project/rules/staging_dedupe.py @@ -23,9 +23,9 @@ def _change_log_sources(ctx: RuleContext) -> set[str]: code="XSQBRDUCK001", message="Staging models over a change-log source must deduplicate it with @sql.cdc_latest", remediation=( - "Read the source in an import CTE and follow it with @sql.cdc_latest(, ), then " - "select from cdc_latest so deletes and superseded versions are handled the same way " - "everywhere." + "Read the source in an import CTE and follow it with " + "@sql.cdc_latest(, src.key := ()), then select from cdc_latest so deletes and " + "superseded versions are handled the same way everywhere." ), ) def staging_uses_cdc_latest(*, model: Model, ctx: RuleContext) -> list[Finding]: From c171063389f355289be2bcca84e998e426699ad2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:36:08 +0000 Subject: [PATCH 4/4] docs(sql-macros): document relation slots --- .../docs/concepts/macros/sql-macros.md | 135 +++++++++++++++++- .../references/macros-and-declarations.md | 14 ++ .../docs/docs/concepts/macros/sql-macros.mdx | 134 ++++++++++++++++- 3 files changed, 275 insertions(+), 8 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index f8dc16a189..d8437e6e70 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/sql-macros/ - Call a SQL macro - Parameter types - Parameters in the body +- Relation slots - Compose macros - Checks - Result types @@ -192,8 +193,7 @@ Every parameter has a type, and may have a `DEFAULT`: | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | -| `RELATION ()` | A CTE name, a table alias, or `__ref()`, `__source()` or `__seed()`, with at least these columns. | -| `COLUMN OF ` | A column name of that relation parameter. | +| `RELATION ()` | A CTE name, a table alias, or `__ref()`, `__source()` or `__seed()`, with the columns its [slots](#relation-slots) read. | SQL types are written as in [schema declarations](../models/schemas.md) and checked against the adapter's SQL dialect the same way, including the dialect's array types such as `VARCHAR[]`. A @@ -269,6 +269,133 @@ error[P023]: places is used where a literal is required; declare it places INT C The positions every adapter shares are checked for every macro file. `DATE_TRUNC` and `ROUND` are checked for the project's adapter at each call. +## Relation slots + +Everything a macro reads from a relation is declared in its `RELATION` type as a slot. A slot is +either one typed column or a `COLUMNS` group of columns the caller chooses, so one macro works at +any grain: + +```sql +-- macros/sql/weekly_rates.sql +MACRO ( + description "Weekly refund rate per grain", + arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)), +); + +WITH counted AS ( + SELECT + s.@weekly.keys, + s.@weekly.order_count AS orders, + s.@weekly.refund_count AS refunds + FROM @weekly AS s +), + +rated AS ( + SELECT + @weekly.keys, + SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate + FROM counted + GROUP BY @weekly.keys +) +``` + +A `COLUMNS` slot is untyped: each column passed to it keeps its own type, so the expanded SQL is +fully typed. + +### Reading slots + +- `alias.@weekly.order_count` reads the slot from a table alias or CTE, such as `s` above. For a + group, the prefix applies to every column: `s.@weekly.keys` becomes `s.product_id, s.region`. +- A plain read of a group, `@weekly.keys`, always stands as the bare column names, + `product_id, region`. So later CTEs, such as `rated` above, read the group from whatever + relation they select from, without an alias. A bare name that two joined relations share fails + as hand-written SQL does, with `B003`; read it as `alias.@weekly.keys` there. +- A plain read of a single-column slot always reads through the passed relation, mapped or not: + `@weekly.refund_count` becomes `weekly.refund_count`, or `weekly.refunds_n` with + `weekly.refund_count := refunds_n`. So the same body expands the same way for every mapping. +- Every read inside `USING (...)` stands unqualified, as `USING` needs, including + `USING (@weekly.week)` for a column the type does not declare. +- In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. + +A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, +could bind its bare columns to the subquery's relations instead of the one the slot reads. The +call fails with `P017` when a relation of an enclosing subquery, other than the relation passed +for the slot, has one of the group's columns, or when its columns are not known. `LATERAL` +subqueries in `FROM` or `JOIN` are checked the same way: + +``` +error[P017]: slot weekly.keys column product_id would bind to refunds in this subquery; write alias.@weekly.keys +``` + +Read the group through its alias there. A group read in a CTE, or in a derived table in `FROM`, +reads that query's own relations and is not checked. + +`alias.@weekly.name` naming no slot, or `alias.@weekly` naming none at all, fails with `P022`. +`@weekly.name` without an alias may still read a column the type does not declare, through the +passed relation, as before. + +### Mapping slots at the call + +A call maps slots with `relation.slot := ...`, next to its other named arguments: + +```sql +WITH weekly AS (SELECT * FROM __ref("weekly_orders")), +@sql.weekly_rates(weekly, weekly.keys := (product_id), weekly.order_count := orders_placed, __cte := by_product) +@sql.weekly_rates(weekly, weekly.keys := (product_id, region), weekly.order_count := orders_placed, __cte := by_product_region) +final AS (SELECT * FROM by_product_region) +SELECT * FROM final +``` + +The first call compiles to the same SQL as a macro written for `product_id`, the second groups by +product and region. The left of `:=` is always a parameter path, and the right side is column +names of the passed relation: + +- A single-column slot takes one column name. An unmapped single-column slot reads the column of + its own name, so `refund_count` above reads `refund_count`. +- A group takes a parenthesised list of column names, such as `(product_id, region)`, and every + call must map it. +- A slot reads a column as it is: compute an expression in the relation you pass. +- A macro that passes its own relation on maps the inner slots to its slot names, as in + `@sql.weekly_rates(weekly := @src, weekly.keys := (dims), weekly.order_count := placed)`, or to + columns of its relation. Each inner slot keeps its kind: a group mapped to columns is still read + bare, or after its alias, and mapping a group to a single-column slot of the outer relation, or + the reverse, fails with `P017`. + +### Rules for groups + +1. A group is pasted verbatim, comma-separated. It takes no per-item expression, indexing, + counting or renaming, and groups are never combined. +2. A group stands only as a whole item of a select list, `PARTITION BY`, `GROUP BY`, `ORDER BY` + (without `ASC`, `DESC` or `NULLS`) or `USING (...)`. Anywhere else fails with `P015` at the + macro file. +3. Grouped columns keep their real names in the output. +4. A group is never empty: `weekly.keys := ()` fails. +5. A relation may have several groups, such as `keys COLUMNS, carried COLUMNS`. Groups of one + relation may not share a column, and a column that fills a single-column slot, mapped or by its + own name, may not also be in a group. + +### Checks at the call + +Slot errors fail with `P017` and name the slot and the caller's relation: + +``` +error[P017]: slot weekly.order_count needs a BIGINT column; weekly has no column orders_total +``` + +- Every mapped and grouped column must exist in the passed relation, and a single-column slot needs + a compatible type: `slot weekly.order_count needs a BIGINT column; weekly.region is VARCHAR`, + or `slot weekly.keys groups column channel; weekly has no column channel`. Like other argument + checks, these run when the relation is a CTE or table alias; the model's own column checks still + run on the expanded SQL. +- Overlapping groups, or a grouped column that also fills a single-column slot, fail at the call. +- A grouped column named like a column the macro defines, or like a bare column such as `s.week` + beside the group in the same select list, fails with `P018`, naming both: + `column refunds of slot weekly.keys clashes with column refunds that macro weekly_rates() defines`. +- A mapped single-column slot read in a select list needs an `AS` in the macro, so output names + don't depend on the caller's mapping. Groups are exempt, because they keep their real names, and + so is an unmapped slot, which keeps its own name. So are reads in `IN`, `EXISTS` and scalar + subqueries, whose column names never reach the macro's output. + ## Compose macros A SQL macro may call other SQL macros, and pass its own parameters on as `@name`: @@ -301,8 +428,8 @@ folder-local macro beside it that its callers cannot see. Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: -- a `RELATION` argument must have every declared column, each of a compatible type; -- a `COLUMN OF` argument must name a column of that relation; +- a `RELATION` argument must have every declared column, each of a compatible type, and every + column its [slots](#checks-at-the-call) are mapped to; - a typed argument must have a compatible type, so `@sql.cents_to_dollars(o.status)` with a `VARCHAR` status fails. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md index 52bb4c35de..297e1b8c0a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/macros-and-declarations.md @@ -42,6 +42,20 @@ known type of another family fails with `P024`. Narrow `ANY` arguments with `ANY `ANY TEXT` or `ANY TEMPORAL`, and call a CTE-list macro more than once in a model with `__cte := name`. +A `RELATION` argument declares everything the macro reads from it as slots: typed single columns +and `COLUMNS` groups of caller-chosen columns. Read them as `@src.slot` or `alias.@src.slot` (a +group's prefix applies to every column) and map them at the call, so one macro serves every grain: + +```sql +-- arguments (weekly RELATION (keys COLUMNS, order_count BIGINT)) +@sql.weekly_rates(w, weekly.keys := (product_id, region), weekly.order_count := orders_placed) +``` + +A group stands only as a whole item of a select list, `PARTITION BY`, `GROUP BY`, `ORDER BY` or +`USING (...)`, and a mapped single-column slot read in a select list needs `AS`. A plain group read +stands as bare column names, so later CTEs need no alias; inside a subquery it fails if a subquery +relation has one of its columns. A plain single-slot read goes through the passed relation. + Full reference: [docs/concepts/macros/sql-macros.md](docs/concepts/macros/sql-macros.md). **Python macros** live in `macros/python/` and are plain Python functions that return final SQL diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 70cba17614..034cd9760f 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -178,8 +178,7 @@ Every parameter has a type, and may have a `DEFAULT`: | ` CONST` | Only a value known at compile time: a literal, a scalar constant, an enum value, or a `CONST` parameter of a calling macro. A column or expression fails with `P017`. Scalar types only. | | `LIST `, `SET ` | An inline list or set, or a list or set constant. Always compile-time, so they take no `CONST`. They render like constants. | | `ENUM ` | A member of that enum, such as `@enum("order_status").OPEN`. | -| `RELATION ()` | A CTE name, a table alias, or `__ref()`, `__source()` or `__seed()`, with at least these columns. | -| `COLUMN OF ` | A column name of that relation parameter. | +| `RELATION ()` | A CTE name, a table alias, or `__ref()`, `__source()` or `__seed()`, with the columns its [slots](#relation-slots) read. | SQL types are written as in [schema declarations](/docs/concepts/models/schemas) and checked against the adapter's SQL dialect the same way, including the dialect's array types such as `VARCHAR[]`. A @@ -255,6 +254,133 @@ error[P023]: places is used where a literal is required; declare it places INT C The positions every adapter shares are checked for every macro file. `DATE_TRUNC` and `ROUND` are checked for the project's adapter at each call. +## Relation slots + +Everything a macro reads from a relation is declared in its `RELATION` type as a slot. A slot is +either one typed column or a `COLUMNS` group of columns the caller chooses, so one macro works at +any grain: + +```sql +-- macros/sql/weekly_rates.sql +MACRO ( + description "Weekly refund rate per grain", + arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)), +); + +WITH counted AS ( + SELECT + s.@weekly.keys, + s.@weekly.order_count AS orders, + s.@weekly.refund_count AS refunds + FROM @weekly AS s +), + +rated AS ( + SELECT + @weekly.keys, + SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate + FROM counted + GROUP BY @weekly.keys +) +``` + +A `COLUMNS` slot is untyped: each column passed to it keeps its own type, so the expanded SQL is +fully typed. + +### Reading slots + +- `alias.@weekly.order_count` reads the slot from a table alias or CTE, such as `s` above. For a + group, the prefix applies to every column: `s.@weekly.keys` becomes `s.product_id, s.region`. +- A plain read of a group, `@weekly.keys`, always stands as the bare column names, + `product_id, region`. So later CTEs, such as `rated` above, read the group from whatever + relation they select from, without an alias. A bare name that two joined relations share fails + as hand-written SQL does, with `B003`; read it as `alias.@weekly.keys` there. +- A plain read of a single-column slot always reads through the passed relation, mapped or not: + `@weekly.refund_count` becomes `weekly.refund_count`, or `weekly.refunds_n` with + `weekly.refund_count := refunds_n`. So the same body expands the same way for every mapping. +- Every read inside `USING (...)` stands unqualified, as `USING` needs, including + `USING (@weekly.week)` for a column the type does not declare. +- In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. + +A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, +could bind its bare columns to the subquery's relations instead of the one the slot reads. The +call fails with `P017` when a relation of an enclosing subquery, other than the relation passed +for the slot, has one of the group's columns, or when its columns are not known. `LATERAL` +subqueries in `FROM` or `JOIN` are checked the same way: + +``` +error[P017]: slot weekly.keys column product_id would bind to refunds in this subquery; write alias.@weekly.keys +``` + +Read the group through its alias there. A group read in a CTE, or in a derived table in `FROM`, +reads that query's own relations and is not checked. + +`alias.@weekly.name` naming no slot, or `alias.@weekly` naming none at all, fails with `P022`. +`@weekly.name` without an alias may still read a column the type does not declare, through the +passed relation, as before. + +### Mapping slots at the call + +A call maps slots with `relation.slot := ...`, next to its other named arguments: + +```sql +WITH weekly AS (SELECT * FROM __ref("weekly_orders")), +@sql.weekly_rates(weekly, weekly.keys := (product_id), weekly.order_count := orders_placed, __cte := by_product) +@sql.weekly_rates(weekly, weekly.keys := (product_id, region), weekly.order_count := orders_placed, __cte := by_product_region) +final AS (SELECT * FROM by_product_region) +SELECT * FROM final +``` + +The first call compiles to the same SQL as a macro written for `product_id`, the second groups by +product and region. The left of `:=` is always a parameter path, and the right side is column +names of the passed relation: + +- A single-column slot takes one column name. An unmapped single-column slot reads the column of + its own name, so `refund_count` above reads `refund_count`. +- A group takes a parenthesised list of column names, such as `(product_id, region)`, and every + call must map it. +- A slot reads a column as it is: compute an expression in the relation you pass. +- A macro that passes its own relation on maps the inner slots to its slot names, as in + `@sql.weekly_rates(weekly := @src, weekly.keys := (dims), weekly.order_count := placed)`, or to + columns of its relation. Each inner slot keeps its kind: a group mapped to columns is still read + bare, or after its alias, and mapping a group to a single-column slot of the outer relation, or + the reverse, fails with `P017`. + +### Rules for groups + +1. A group is pasted verbatim, comma-separated. It takes no per-item expression, indexing, + counting or renaming, and groups are never combined. +2. A group stands only as a whole item of a select list, `PARTITION BY`, `GROUP BY`, `ORDER BY` + (without `ASC`, `DESC` or `NULLS`) or `USING (...)`. Anywhere else fails with `P015` at the + macro file. +3. Grouped columns keep their real names in the output. +4. A group is never empty: `weekly.keys := ()` fails. +5. A relation may have several groups, such as `keys COLUMNS, carried COLUMNS`. Groups of one + relation may not share a column, and a column that fills a single-column slot, mapped or by its + own name, may not also be in a group. + +### Checks at the call + +Slot errors fail with `P017` and name the slot and the caller's relation: + +``` +error[P017]: slot weekly.order_count needs a BIGINT column; weekly has no column orders_total +``` + +- Every mapped and grouped column must exist in the passed relation, and a single-column slot needs + a compatible type: `slot weekly.order_count needs a BIGINT column; weekly.region is VARCHAR`, + or `slot weekly.keys groups column channel; weekly has no column channel`. Like other argument + checks, these run when the relation is a CTE or table alias; the model's own column checks still + run on the expanded SQL. +- Overlapping groups, or a grouped column that also fills a single-column slot, fail at the call. +- A grouped column named like a column the macro defines, or like a bare column such as `s.week` + beside the group in the same select list, fails with `P018`, naming both: + `column refunds of slot weekly.keys clashes with column refunds that macro weekly_rates() defines`. +- A mapped single-column slot read in a select list needs an `AS` in the macro, so output names + don't depend on the caller's mapping. Groups are exempt, because they keep their real names, and + so is an unmapped slot, which keeps its own name. So are reads in `IN`, `EXISTS` and scalar + subqueries, whose column names never reach the macro's output. + ## Compose macros A SQL macro may call other SQL macros, and pass its own parameters on as `@name`: @@ -287,8 +413,8 @@ folder-local macro beside it that its callers cannot see. Each macro is checked on its own when the project compiles, and each call is checked against the caller's column types, inferred like the rest of the model: -- a `RELATION` argument must have every declared column, each of a compatible type; -- a `COLUMN OF` argument must name a column of that relation; +- a `RELATION` argument must have every declared column, each of a compatible type, and every + column its [slots](#checks-at-the-call) are mapped to; - a typed argument must have a compatible type, so `@sql.cents_to_dollars(o.status)` with a `VARCHAR` status fails.