From 294fd80f1c28150c9f95a70d2c0d198574c3aede Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:59:48 +0100 Subject: [PATCH 1/2] fix: lint SQL macro bodies as compiled and pass SQL macro expansions to Python macros intact Linting a SQL macro file read a parameter followed by words and a parenthesis, such as `@value IS NULL OR TRIM(@value) = ''`, as an unprefixed macro call (L005), and a body opening with two parentheses reached the native parser as a statement it rejected (rules-unevaluated). - One rule decides an unprefixed call everywhere: `@name(` with the parenthesis right after the name. The Python macro call scanner, the SQL macro lexer, parameter references and the bare-call check share it, so `@name IS NULL OR f(x)` in model SQL is no longer P014 either. `@const`, `@enum` and `@param` stay declarations with or without a space before their parenthesis. - Lint replaces a SQL macro body's parameters by sentinels with the compiler's own parameter scan, lints an expression body as the `SELECT` the compiler parses it as, and a CTE-list body with its ceremonial `SELECT 1` and its CTEs counted as used by the caller. Findings in the generated frame are dropped. - A SQL macro expansion passed to a Python macro is spliced as a string literal with `"` written as `\x22`, which the call scanner and the argument parser read the same way, so quotes, backslashes, line breaks and comments reach the macro unchanged. - An expansion ending in a line comment gets a line break only when text would follow it on the same line, so the caller's text stays code and calls already last on their line compile unchanged. - A macro call the scan cannot read is a MacroArgumentError naming the call, file, line and column instead of a location-less P001. - Rules on a caller skip comments a SQL macro expansion generated; the macro file is where they are judged, and compiled SQL keeps them. Comments from Python macro output are judged as before. --- .../src/sql_macros/_helpers/body.rs | 33 ++ .../src/sql_macros/_helpers/definition.rs | 2 +- .../src/sql_macros/_helpers/expansion.rs | 41 +- .../src/sql_macros/_helpers/lexing.rs | 6 +- .../src/sql_macros/_helpers/parameters.rs | 10 +- .../src/sql_macros/main/bare_macro_calls.rs | 8 +- .../src/sql_macros/main/mod.rs | 2 + .../sql_macros/main/sql_macro_body_form.rs | 14 + .../main/sql_macro_parameter_spans.rs | 16 + .../src/sql_macros/tests/helpers.rs | 12 +- .../src/sql_macros/tests/test_expansion.rs | 208 ++++++++- .../src/sql_macros/tests/test_types.rs | 16 + .../_helpers/record_behaviours/error_inits.rs | 12 +- .../_helpers/render/macro_evaluation.rs | 68 +-- .../_helpers/render/macro_expansion.rs | 54 ++- .../bindings/_helpers/rules/lint_pipeline.rs | 88 +++- .../_helpers/rules/lint_sql_bodies.rs | 1 + .../src/macro_calls/_helpers/scanner.rs | 9 +- .../src/macro_calls/main/mod.rs | 1 + .../src/macro_calls/main/opens_bare_call.rs | 7 + .../tests/test_scan_macro_call_sites.rs | 12 + .../src/lint_text/_helpers/sql_diagnostics.rs | 85 +++- .../src/lint_text/_helpers/sql_requests.rs | 8 +- .../sqlbuild-rules/src/lint_text/constants.rs | 3 + .../main/frame_sql_macro_expression.rs | 39 ++ .../src/lint_text/main/lint_sql_bodies.rs | 17 +- .../sqlbuild-rules/src/lint_text/main/mod.rs | 2 + .../main/neutralize_sql_macro_parameters.rs | 23 + crates/sqlbuild-rules/src/lint_text/models.rs | 11 + .../src/lint_text/tests/test_lint_text.rs | 133 ++++++ .../src/sql_lint/_helpers/preparation.rs | 15 + .../src/sql_lint/main/defined_cte_names.rs | 7 + .../sqlbuild-rules/src/sql_lint/main/mod.rs | 1 + .../main/compile/sql_macros/_test_types.py | 12 + .../main/compile/sql_macros/helpers.py | 40 ++ .../test_sql_macro_expansion_text.py | 430 ++++++++++++++++++ ..._expansion_nested_argument_scan_edges.json | 14 +- .../compile/test_native_macro_bridge.py | 2 +- 38 files changed, 1360 insertions(+), 102 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs create mode 100644 crates/sqlbuild-render/src/macro_calls/main/opens_bare_call.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/frame_sql_macro_expression.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/neutralize_sql_macro_parameters.rs create mode 100644 crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs index 238aca8e35..69de0a50a4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/body.rs @@ -295,6 +295,39 @@ pub(crate) fn expand_body( Ok(BodyExpansion { sql, names }) } +/// `sql`, with a line break when it ends in a `--` comment and `followed` text would join that line. +pub(crate) fn with_line_comment_closed(sql: &str, followed: bool) -> String { + let mut sql: String = sql.to_owned(); + if !followed { + return sql; + } + let Ok(scanned) = tokens(&sql) else { + return sql; + }; + let mut rest: &str = &sql[scanned.last().map_or(0, |last| last.end)..]; + let mut open_line_comment: bool = false; + loop { + rest = rest.trim_start(); + if let Some(comment) = rest.strip_prefix("--") { + match comment.split_once('\n') { + Some((_, after)) => rest = after, + None => { + open_line_comment = true; + break; + } + } + } else if let Some(comment) = rest.strip_prefix("/*") { + rest = comment.split_once("*/").map_or("", |(_, after)| after); + } else { + break; + } + } + if open_line_comment { + sql.push('\n'); + } + sql +} + /// The text after its first token, the body's `WITH` or `SELECT`, trimmed. fn without_leading_keyword(text: &str) -> String { match tokens(text) { diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs index d0affd9528..0631d5a5af 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/definition.rs @@ -106,7 +106,7 @@ fn unclosed_body(name: &str, location: SqlMacroLocation) -> SqlMacroError { } /// The form a body's first token decides. -fn body_form(body: &str, scanned: &[Token]) -> SqlMacroForm { +pub(crate) fn body_form(body: &str, scanned: &[Token]) -> SqlMacroForm { match scanned.first() { Some(first) if first.is_word(body, WITH_KEYWORD) => SqlMacroForm::CteList, Some(first) if first.is_word(body, SELECT_KEYWORD) => SqlMacroForm::ColumnList, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs index 224339a2c0..6962997c11 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/expansion.rs @@ -6,7 +6,7 @@ use crate::compiler::main::function_type_validation::function_type_error; use crate::sql_macros::_helpers::arguments::{ ShapeContext, argument_shape, bind_arguments, check_shape, is_atomic, split_arguments, }; -use crate::sql_macros::_helpers::body::expand_body; +use crate::sql_macros::_helpers::body::{expand_body, with_line_comment_closed}; use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs, outer_sql_calls}; use crate::sql_macros::_helpers::lexing::{Token, line_column, tokens}; use crate::sql_macros::_helpers::literal_positions::{ @@ -209,7 +209,10 @@ pub(crate) fn expand_call( let expansion: BodyExpansion = expand_body(&definition, &body, &rendered) .map_err(|error| error.at(body_location(&definition)))?; if call.in_python { - return Ok((python_string(&expansion.sql), inserted)); + return Ok(( + python_string(&with_line_comment_closed(&expansion.sql, true)), + inserted, + )); } if matches!(context.scope, SqlMacroScope::Caller) && context.stack.is_empty() { let check: SqlMacroCheck = SqlMacroCheck { @@ -229,8 +232,17 @@ pub(crate) fn expand_call( (call.start, call.end), call.replaced_end, ); - placed(&definition, (&site, inserted), &expansion, context.owner) - .map_err(ExpansionFailure::Macro) + let line_continues: bool = context.text[call.replaced_end..] + .split('\n') + .next() + .is_some_and(|rest| !rest.trim().is_empty()); + placed( + &definition, + (&site, inserted, line_continues), + &expansion, + context.owner, + ) + .map_err(ExpansionFailure::Macro) } /// One bound argument checked against its parameter and rendered from `scope`. @@ -306,14 +318,14 @@ fn rendered_argument( }) } -/// `sql` as a double-quoted Python string literal. +/// `sql` as a double-quoted Python string literal holding no `"` and no line break. fn python_string(sql: &str) -> String { let mut quoted: String = String::with_capacity(sql.len() + 2); quoted.push('"'); for character in sql.chars() { match character { '\\' => quoted.push_str("\\\\"), - '"' => quoted.push_str("\\\""), + '"' => quoted.push_str("\\x22"), '\n' => quoted.push_str("\\n"), '\r' => quoted.push_str("\\r"), '\t' => quoted.push_str("\\t"), @@ -327,10 +339,17 @@ fn python_string(sql: &str) -> String { /// The expansion placed at the call, or why it cannot stand there or clashes. fn placed( definition: &SqlMacroDefinition, - (site, inserted): (&CallSite<'_>, InsertedNames), + (site, inserted, line_continues): (&CallSite<'_>, InsertedNames, bool), expansion: &BodyExpansion, owner: &str, ) -> Result<(String, InsertedNames), SqlMacroError> { + let closed = |after: &str| { + let followed: bool = match after.chars().next() { + None => line_continues, + Some(next) => next != '\n' && next != '\r', + }; + with_line_comment_closed(&expansion.sql, followed) + }; let position: CallPosition = site.position(); let name: &str = &definition.name; match definition.form { @@ -341,9 +360,9 @@ fn placed( } let comma: &str = site.expression_separator(); if site.expression_needs_parentheses() && !is_atomic(&expansion.sql) { - Ok((format!("({}){comma}", expansion.sql), inserted)) + Ok((format!("({}){comma}", closed(")")), inserted)) } else { - Ok((format!("{}{comma}", expansion.sql), inserted)) + Ok((format!("{}{comma}", closed(comma)), inserted)) } } SqlMacroForm::CteList => { @@ -365,7 +384,7 @@ fn placed( ("CTE", owner), )?; let (before, after) = site.cte_separators(); - Ok((format!("{before}{}{after}", expansion.sql), inserted)) + Ok((format!("{before}{}{after}", closed(after)), inserted)) } SqlMacroForm::ColumnList => { if position != CallPosition::SelectList { @@ -395,7 +414,7 @@ fn placed( ("column", owner), )?; let (before, after) = site.column_separators(); - Ok((format!("{before}{}{after}", expansion.sql), inserted)) + Ok((format!("{before}{}{after}", closed(after)), inserted)) } } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs index 4f70fb2a7d..d17c54dddc 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/lexing.rs @@ -1,5 +1,7 @@ //! A comment- and quote-aware SQL token scan, and the `@...` constructs it finds. +use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; + use crate::sql_macros::constants::SQL_KIND_WORD; use crate::sql_macros::errors::LexError; @@ -45,7 +47,7 @@ impl Token { const PUNCTUATION: &[u8] = b"()[]{},.;"; /// The `@word(...)` declaration forms that are not macro calls. -const DECLARATION_WORDS: [&str; 3] = ["const", "enum", "param"]; +pub(crate) const DECLARATION_WORDS: [&str; 3] = ["const", "enum", "param"]; /// Every token of `sql`; text unclosed under doubled-quote escapes is reread with backslash ones. pub(crate) fn tokens(sql: &str) -> Result, LexError> { @@ -341,7 +343,7 @@ pub(crate) fn at_constructs(sql: &str) -> Result, LexError> { }); continue; } - if scanned.get(index + 2).is_some_and(|open| open.is(sql, "(")) { + if opens_bare_call(sql, word.end) { found.push(AtConstruct::Bare { name: word_text.to_owned(), start: token.start, diff --git a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs index 426c254792..05039f6b00 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/_helpers/parameters.rs @@ -1,6 +1,8 @@ //! `@name` parameter references in a SQL macro body, as generic audits write their parameters. -use crate::sql_macros::_helpers::lexing::{Token, TokenKind}; +use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; + +use crate::sql_macros::_helpers::lexing::{DECLARATION_WORDS, Token, TokenKind}; const PLACEHOLDER_PREFIX: &str = "__sqlbuild_parameter__"; /// Words that may follow a table reference without being its alias. @@ -68,10 +70,12 @@ pub(crate) fn parameter_references(sql: &str, scanned: &[Token]) -> Vec = scanned.get(index + 2); - if after.is_some_and(|open| open.is(sql, "(")) { + let name: &str = word.text(sql); + let declaration: bool = + DECLARATION_WORDS.contains(&name) && after.is_some_and(|open| open.is(sql, "(")); + if declaration || opens_bare_call(sql, word.end) { continue; } - let name: &str = word.text(sql); if matches!(name, "sql" | "py") && after.is_some_and(|dot| dot.is(sql, ".") && dot.start == word.end) && scanned.get(index + 4).is_some_and(|open| open.is(sql, "(")) diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs index b57adfd90e..32277f5163 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/bare_macro_calls.rs @@ -1,10 +1,11 @@ use crate::sql_macros::_helpers::lexing::{AtConstruct, at_constructs}; use crate::sql_macros::constants::NON_ASCII_BYTE; +use sqlbuild_render::macro_calls::main::opens_bare_call::opens_bare_call; /// The name and byte offset of each `@name(...)` call in `text` that names no macro kind. #[must_use] pub fn bare_macro_calls(text: &str) -> Vec<(String, usize)> { - if !may_hold_bare_call(text.as_bytes()) { + if !may_hold_bare_call(text) { return Vec::new(); } at_constructs(text) @@ -18,7 +19,8 @@ pub fn bare_macro_calls(text: &str) -> Vec<(String, usize)> { } /// Whether an `@` followed by a name and `(` that is not a kind prefix or declaration occurs. -fn may_hold_bare_call(bytes: &[u8]) -> bool { +fn may_hold_bare_call(text: &str) -> bool { + let bytes: &[u8] = text.as_bytes(); for at in memchr::memchr_iter(b'@', bytes) { let start: usize = at + 1; let mut end: usize = start; @@ -36,7 +38,7 @@ fn may_hold_bare_call(bytes: &[u8]) -> bool { ) { continue; } - if bytes.get(end) == Some(&b'(') { + if opens_bare_call(text, end) { return true; } } diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs index 66d5595ff7..38f29462c4 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/main/mod.rs @@ -4,6 +4,8 @@ pub mod declaration_visible_from_macro; pub mod expand_sql_macro_call; pub mod parse_sql_macro; pub mod render_inline_collection; +pub mod sql_macro_body_form; pub mod sql_macro_calls; pub mod sql_macro_cycle; +pub mod sql_macro_parameter_spans; pub mod with_list_sql_macro_calls; diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs new file mode 100644 index 0000000000..62db814bb3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_body_form.rs @@ -0,0 +1,14 @@ +//! The form of a SQL macro body, as its definition reads it. + +use crate::sql_macros::_helpers::definition::body_form; +use crate::sql_macros::_helpers::lexing::tokens; +use crate::sql_macros::models::SqlMacroForm; + +/// Whether `body` is a CTE list, a column list or an expression, from its first token. +#[must_use] +pub fn sql_macro_body_form(body: &str) -> SqlMacroForm { + match tokens(body) { + Ok(scanned) => body_form(body, &scanned), + Err(_) => SqlMacroForm::Expression, + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs new file mode 100644 index 0000000000..419d4fcaff --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/main/sql_macro_parameter_spans.rs @@ -0,0 +1,16 @@ +//! The byte spans of a SQL macro body's `@name` parameter references, as expansion reads them. + +use crate::sql_macros::_helpers::lexing::tokens; +use crate::sql_macros::_helpers::parameters::parameter_references; + +/// The `start..end` byte span of each `@name` in `body` that expansion replaces by an argument. +#[must_use] +pub fn sql_macro_parameter_spans(body: &str) -> Vec<(usize, usize)> { + let Ok(scanned) = tokens(body) else { + return Vec::new(); + }; + parameter_references(body, &scanned) + .into_iter() + .map(|reference| (reference.start, reference.end)) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index 25d63ce286..8763b5a0ab 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -5,7 +5,7 @@ use crate::sql_macros::errors::ExpansionFailure; use crate::sql_macros::errors::SqlMacroError; use crate::sql_macros::main::parse_sql_macro::parse_sql_macro; use crate::sql_macros::models::{SqlMacroDefinition, TextContext}; -use crate::sql_macros::tests::test_types::FakeHost; +use crate::sql_macros::tests::test_types::{AsciiArguments, FakeHost}; use crate::sql_macros::types::{ConstantShape, SqlMacroHost, SqlMacroScope}; /// Constants and enum members the fake host renders, by reference text. @@ -116,3 +116,13 @@ fn macro_error(failure: ExpansionFailure) -> SqlMacroError { pub(super) fn owned(names: &[&str]) -> Vec { names.iter().map(|name| (*name).to_owned()).collect() } + +impl sqlbuild_render::macro_arguments::types::ArgumentHost for AsciiArguments { + fn character_named(&self, _name: &str) -> Option { + None + } + + fn identifier(&self, _text: &str) -> Option { + None + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs index 2000056e52..167a3f279f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_expansion.rs @@ -1,5 +1,11 @@ +use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments; +use sqlbuild_render::macro_arguments::models::ArgumentValue; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites; + use crate::sql_macros::tests::helpers::expanded_for; -use crate::sql_macros::tests::test_types::{ExpansionErrorTestCase, ExpansionTestCase}; +use crate::sql_macros::tests::test_types::{ + AsciiArguments, ExpansionErrorTestCase, ExpansionTestCase, PythonArgumentTestCase, +}; const CENTS_TO_DOLLARS: (&str, &str) = ( "cents_to_dollars", @@ -87,6 +93,30 @@ const HAS_TAG: (&str, &str) = ( "has_tag", "MACRO (description \"Tagged\", arguments (tags VARCHAR[], tag VARCHAR), returns BOOLEAN);\n\nlist_contains(@tags, @tag)\n", ); +const IS_BLANK: (&str, &str) = ( + "is_blank", + "MACRO (description \"Blank\", arguments (value VARCHAR), returns BOOLEAN);\n\n@value IS NULL OR TRIM(@value) = ''\n", +); +const IS_BLANK_PARENTHESISED: (&str, &str) = ( + "is_blank", + "MACRO (description \"Blank\", arguments (value VARCHAR), returns BOOLEAN);\n\n((@value) IS NULL AND (@value) IS NULL) OR TRIM(@value) = ''\n", +); +const BOTH_MISSING: (&str, &str) = ( + "both_missing", + "MACRO (description \"Both missing\", arguments (x VARCHAR, y VARCHAR), returns BOOLEAN);\n\n@x IS NULL AND (@y IS NULL)\n", +); +const IS_LETTER: (&str, &str) = ( + "is_letter", + "MACRO (description \"Letter\", arguments (x VARCHAR), returns BOOLEAN);\n\nCASE WHEN @x IN ('a', 'b') THEN TRUE ELSE FALSE END\n", +); +const TRAILING_COMMENT: (&str, &str) = ( + "has_x", + "MACRO (description \"Has x\", arguments (value VARCHAR), returns BOOLEAN);\n\nCONTAINS(@value, 'x') -- why x\n", +); +const SPACED_DECLARATIONS: (&str, &str) = ( + "spaced", + "MACRO (description \"Spaced\", arguments (status VARCHAR, places INT), returns BOOLEAN);\n\n@status <> @enum (\"order_status\").CLOSED AND @places = @const (\"price_places\")\n", +); const ODD: (&str, &str) = ( "odd", "MACRO (description \"Odd\", arguments (v FOO BAR), returns BIGINT);\n\n@v + 1\n", @@ -284,6 +314,62 @@ fn given_calls_of_each_form_when_expanding_then_the_ticket_examples_compile() { sql: "SELECT @sql.has_tag(o.tags, 'vip') AS vip FROM orders AS o", expected_sql: "SELECT list_contains(o.tags, 'vip') AS vip FROM orders AS o", }, + ExpansionTestCase { + description: "a parameter followed by words and a parenthesis is a parameter", + adapter: "", + macros: &[IS_BLANK], + sql: "SELECT @sql.is_blank(o.email) AS b FROM orders AS o", + expected_sql: "SELECT o.email IS NULL OR TRIM(o.email) = '' AS b FROM orders AS o", + }, + ExpansionTestCase { + description: "parenthesised parameters in a body opening with two parentheses", + adapter: "", + macros: &[IS_BLANK_PARENTHESISED], + sql: "SELECT @sql.is_blank(o.email) AS b FROM orders AS o", + expected_sql: "SELECT ((o.email) IS NULL AND (o.email) IS NULL) OR TRIM(o.email) = '' AS b FROM orders AS o", + }, + ExpansionTestCase { + description: "a parameter before AND and a parenthesised predicate", + adapter: "", + macros: &[BOTH_MISSING], + sql: "SELECT @sql.both_missing(o.a, o.b) AS m FROM orders AS o", + expected_sql: "SELECT o.a IS NULL AND (o.b IS NULL) AS m FROM orders AS o", + }, + ExpansionTestCase { + description: "a parameter before IN inside a CASE", + adapter: "", + macros: &[IS_LETTER], + sql: "SELECT @sql.is_letter(o.a) AS l FROM orders AS o", + expected_sql: "SELECT CASE WHEN o.a IN ('a', 'b') THEN TRUE ELSE FALSE END AS l FROM orders AS o", + }, + ExpansionTestCase { + description: "a body ending in a line comment ends its line, so the call's alias stays code", + adapter: "", + macros: &[TRAILING_COMMENT], + sql: "SELECT @sql.has_x(o.a) AS l FROM orders AS o", + expected_sql: "SELECT CONTAINS(o.a, 'x') -- why x\n AS l FROM orders AS o", + }, + ExpansionTestCase { + description: "a body ending in a line comment, called last on its line, adds no line", + adapter: "", + macros: &[TRAILING_COMMENT], + sql: "SELECT @sql.has_x(o.a)\nFROM orders AS o", + expected_sql: "SELECT CONTAINS(o.a, 'x') -- why x\nFROM orders AS o", + }, + ExpansionTestCase { + description: "a body ending in a line comment, followed by an operator, closes its line", + adapter: "", + macros: &[TRAILING_COMMENT], + sql: "SELECT @sql.has_x(o.a) = TRUE\nFROM orders AS o", + expected_sql: "SELECT CONTAINS(o.a, 'x') -- why x\n = TRUE\nFROM orders AS o", + }, + ExpansionTestCase { + description: "spaced enum and constant references in a body are declarations", + adapter: "", + macros: &[SPACED_DECLARATIONS], + sql: "SELECT @sql.spaced(o.status, o.places) AS s FROM orders AS o", + expected_sql: "SELECT o.status <> @enum (\"order_status\").CLOSED AND o.places = @const (\"price_places\") AS s FROM orders AS o", + }, ExpansionTestCase { description: "generic SQL checks no adapter types", adapter: "", @@ -463,3 +549,123 @@ fn given_invalid_calls_when_expanding_then_each_error_names_the_problem() { ); } } + +#[test] +fn given_a_sql_macro_call_in_python_macro_arguments_when_expanding_then_python_receives_its_expansion() + { + let wrapped = |sql: String| vec![ArgumentValue::Str(sql)]; + let test_cases = [ + PythonArgumentTestCase { + description: "a double quote inside a SQL string literal", + body: "CONTAINS(@value, '\"')", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "a double quote compared directly", + body: "@value = '\"'", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "a single quote inside a double-quoted identifier", + body: "@value = \"it's\"", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "doubled single quotes", + body: "@value = ''''", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "backslashes, one before a quote", + body: "@value LIKE 'a\\_b' OR @value = '\\'", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "a line break", + body: "@value = 'a'\n OR @value = 'b'", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "a line comment, a block comment and a comment ending the body", + body: "-- why\nCONTAINS(@value, /* sign */ '\"') -- end", + sql: "SELECT @py.wrap(@sql.m(o.email)) AS probe", + expected_suffix: "\n", + expected_arguments: wrapped, + }, + PythonArgumentTestCase { + description: "a dict value", + body: "CONTAINS(@value, '\"')", + sql: "SELECT @py.pick({\"a\": @sql.m(o.email)}) AS probe", + expected_suffix: "", + expected_arguments: |sql| { + vec![ArgumentValue::Dict(vec![( + ArgumentValue::Str("a".to_owned()), + ArgumentValue::Str(sql), + )])] + }, + }, + PythonArgumentTestCase { + description: "a nested list", + body: "CONTAINS(@value, '\"')", + sql: "SELECT @py.pick([@sql.m(o.email), [@sql.m(o.email)]]) AS probe", + expected_suffix: "", + expected_arguments: |sql| { + vec![ArgumentValue::List(vec![ + ArgumentValue::Str(sql.clone()), + ArgumentValue::List(vec![ArgumentValue::Str(sql)]), + ])] + }, + }, + ]; + + for test_case in test_cases { + let source: String = format!( + "MACRO (description \"M\", arguments (value VARCHAR), returns BOOLEAN);\n\n{}\n", + test_case.body + ); + let macros: [(&str, &str); 1] = [("m", source.as_str())]; + let direct: Result = expanded_for("", ¯os, "@sql.m(o.email)"); + let expanded: Result = expanded_for("", ¯os, test_case.sql); + let (Ok(direct), Ok(expanded)) = (direct, expanded) else { + panic!("{} does not expand", test_case.description); + }; + + let scan = scan_macro_call_sites(&expanded); + let arguments: Option = scan.sites.first().map(|site| { + expanded + .chars() + .skip(site.args_start) + .take(site.end - 1 - site.args_start) + .collect() + }); + let parsed = + arguments.map(|arguments| parse_macro_arguments(&AsciiArguments, &arguments, &[])); + + assert_eq!( + (scan.failure, scan.sites.len()), + (None, 1), + "{}", + test_case.description + ); + assert_eq!( + parsed.map(|parsed| parsed.map(|arguments| arguments.positional)), + Some(Ok((test_case.expected_arguments)( + direct + test_case.expected_suffix + ))), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index cff783d1d2..493f530145 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -1,6 +1,8 @@ use std::collections::HashMap; use std::sync::Arc; +use sqlbuild_render::macro_arguments::models::ArgumentValue; + use crate::sql_macros::models::SqlMacroDefinition; pub(super) struct ExpansionTestCase { @@ -46,3 +48,17 @@ pub(super) struct FakeHost { pub(super) definitions: HashMap>, pub(super) adapter: &'static str, } + +/// A model calling a Python macro with a SQL macro call among its arguments. +pub(super) struct PythonArgumentTestCase { + pub(super) description: &'static str, + pub(super) body: &'static str, + pub(super) sql: &'static str, + /// What a Python macro receives after the direct expansion: a line break after a `--` comment. + pub(super) expected_suffix: &'static str, + /// The argument values, given the SQL macro call's own expansion. + pub(super) expected_arguments: fn(String) -> Vec, +} + +/// Argument parsing without Unicode name lookups, which these arguments never use. +pub(super) struct AsciiArguments; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs index c6a35ce2ab..bf3aeb9fce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs @@ -68,16 +68,18 @@ pub(crate) fn macro_argument_error_init( native_field(&location, "column")?, ) }; + let call: String = if macro_name.starts_with('@') { + macro_name + } else { + format!("@py.{macro_name}") + }; let (scope, suffix): (String, String) = if location.is_none() { ( format!("in '{file_path}'"), - format!(" of the '@py.{macro_name}' arguments"), + format!(" of the '{call}' arguments"), ) } else { - ( - format!("of '@py.{macro_name}' in '{file_path}'"), - String::new(), - ) + (format!("of '{call}' in '{file_path}'"), String::new()) }; let message: String = format!( "Macro arguments {scope} {detail} at line {}, column {}{suffix}", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs index be992c85d0..b4da44c688 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs @@ -261,7 +261,9 @@ fn call_arguments<'py>( values.push(evaluate_call(run, (&text, site), false)?); } if let Some(failure) = &scan.failure { - return Err(run.expansion.failed_scan(run.classes, failure, py)); + return Err(run + .expansion + .failed_scan(run.classes, (&text, failure), Some(name), py)); } let nested: Vec<(usize, usize)> = scan .sites @@ -399,31 +401,43 @@ impl<'py> ArgumentSource<'_, 'py> { /// `MacroArgumentError` at a code point offset of the argument text. fn argument_error(&self, offset: usize, detail: &str, help: &str) -> PyErr { - let before: &str = self.text.slice(0, offset.min(self.text.bytes.len() - 1)); - let line: usize = before.matches('\n').count() + 1; - let column: usize = before - .rsplit_once('\n') - .map_or(before, |(_, last)| last) - .chars() - .count() - + 1; - let created = (|| { - let py: Python<'_> = self.run.handle.py(); - let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("detail", detail)?; - fields.set_item("help", help)?; - fields.set_item("macro_name", self.name)?; - fields.set_item("file_path", self.run.expansion.file_path.bind(py))?; - fields.set_item("offset", offset)?; - fields.set_item("relative_position", (line, column))?; - self.run - .classes - .macro_argument_error - .call((), Some(&fields)) - })(); - match created { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } + macro_argument_error( + (self.run.expansion, self.run.classes), + self.text, + (self.name, offset), + (detail, help), + ) + } +} + +/// `MacroArgumentError` of the call `name` at code point `offset` of `text`. +pub(crate) fn macro_argument_error( + (expansion, classes): (&NativeMacroExpansion, &ExpansionClasses<'_>), + text: &CallText<'_>, + (name, offset): (&str, usize), + (detail, help): (&str, &str), +) -> PyErr { + let before: &str = text.slice(0, offset.min(text.bytes.len() - 1)); + let line: usize = before.matches('\n').count() + 1; + let column: usize = before + .rsplit_once('\n') + .map_or(before, |(_, last)| last) + .chars() + .count() + + 1; + let created = (|| { + let py: Python<'_> = classes.macro_argument_error.py(); + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("detail", detail)?; + fields.set_item("help", help)?; + fields.set_item("macro_name", name)?; + fields.set_item("file_path", expansion.file_path.bind(py))?; + fields.set_item("offset", offset)?; + fields.set_item("relative_position", (line, column))?; + classes.macro_argument_error.call((), Some(&fields)) + })(); + match created { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index 21409de3b6..5065fec0ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -24,7 +24,9 @@ use crate::bindings::_helpers::render::declaration_expansion::usage_visibility; use crate::bindings::_helpers::render::macro_call_classes::{CallClass, ClassInputs}; use crate::bindings::_helpers::render::macro_call_memo::{MacroCallMemo, poisoned}; use crate::bindings::_helpers::render::macro_call_rows::{SpanTuple, span_row, span_tuple}; -use crate::bindings::_helpers::render::macro_evaluation::{CallText, Evaluation, evaluate_call}; +use crate::bindings::_helpers::render::macro_evaluation::{ + CallText, Evaluation, evaluate_call, macro_argument_error, +}; use crate::bindings::_helpers::render::macro_expansion_facts::{ExpansionFacts, ResourceRef}; use crate::bindings::_helpers::render::macro_generated_references::{ GeneratedOrigin, GeneratedReferenceClasses, report_generated_references, @@ -37,6 +39,32 @@ const RELATION_SUFFIX: &str = "__"; const CONSTANT_KIND: &str = "constant"; const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; const SQL_LANGUAGE: &str = "sql"; +const DECLARATION_FORMS: [&str; 3] = ["const", "enum", "param"]; +const UNREAD_CALL_HELP: &str = "Close every quote, block comment and parenthesis in the macro \ + call, and follow its name with its opening parenthesis"; + +/// Why the scan could not read the call `@{called}`, if the scan stopped inside a call. +fn unread_call_detail(error: ScanError, called: &str) -> Option { + let reason: String = match error { + ScanError::UnclosedQuote => format!("a quoted string in {called} is not closed"), + ScanError::UnclosedComment => format!("a block comment in {called} is not closed"), + ScanError::UnclosedParenthesis => format!("{called} is not closed with ')'"), + ScanError::MissingParenthesis => { + format!("{called} is not followed by its opening parenthesis") + } + ScanError::BareCall | ScanError::SqlCall => return None, + }; + Some(format!("could not be read: {reason}")) +} + +/// How a scanned call is written: `@py.name`, or `@const` and the other declaration forms. +fn written_call(name: &str) -> String { + if DECLARATION_FORMS.contains(&name) { + format!("@{name}") + } else { + format!("@py.{name}") + } +} positional_record! { into; @@ -293,7 +321,7 @@ impl NativeMacroExpansion { outputs.push(self.evaluated_output(run, (text, site))?); } if let Some(failure) = &scan.failure { - return Err(self.failed_scan(run.classes, failure, run.handle.py())); + return Err(self.failed_scan(run.classes, (text, failure), None, run.handle.py())); } Ok(outputs) } @@ -318,11 +346,12 @@ impl NativeMacroExpansion { Ok(false) } - /// `raise_failed_macro_call`: the failing call's macro lookup error, else the scan's error. + /// `raise_failed_macro_call`: the lookup error, the unreadable call located, or the scan's error. pub(crate) fn failed_scan( &self, classes: &ExpansionClasses<'_>, - failure: &MacroScanFailure, + (text, failure): (&CallText<'_>, &MacroScanFailure), + enclosing: Option<&str>, py: Python<'_>, ) -> PyErr { let name: &str = failure.call_name.as_deref().unwrap_or_default(); @@ -359,6 +388,23 @@ impl NativeMacroExpansion { { return error; } + let written: String = written_call(name); + if let (Some(start), Some(detail)) = ( + failure.call_start, + unread_call_detail(failure.error, &written), + ) { + let labelled: &str = match enclosing { + Some(enclosing) => enclosing, + None if DECLARATION_FORMS.contains(&name) => &written, + None => name, + }; + return macro_argument_error( + (self, classes), + text, + (labelled, start), + (&detail, UNREAD_CALL_HELP), + ); + } raised(&classes.compile_error, failure.error.message()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index c6b4431710..1eac80974b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -8,6 +8,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_analysis::sql_macros::main::sql_macro_body_form::sql_macro_body_form; +use sqlbuild_analysis::sql_macros::models::SqlMacroForm; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -21,17 +23,20 @@ use sqlbuild_rules::dsl_headers::models::{ use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; +use sqlbuild_rules::lint_text::main::frame_sql_macro_expression::frame_sql_macro_expression; use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind; use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; +use sqlbuild_rules::lint_text::main::neutralize_sql_macro_parameters::neutralize_sql_macro_parameters; use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::models::{ - AuthoredDiagnostic, ExpansionSpan, InterpolationSite, LintSqlBody, LintSqlOptions, Neutralized, - SuppressionFinding, + AuthoredDiagnostic, ExpansionSpan, FramedExpression, InterpolationSite, LintSqlBody, + LintSqlOptions, Neutralized, SuppressionFinding, }; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::project_tree::main::project_real_paths::project_real_paths; +use sqlbuild_rules::sql_lint::main::defined_cte_names::defined_cte_names; use sqlbuild_rules::sql_lint::main::fill_finding_path::fill_finding_path; use sqlbuild_rules::sql_lint::main::preparation::prepare; use sqlbuild_rules::sql_lint::types::PreparedSql; @@ -58,6 +63,7 @@ const AUDIT_KIND: &str = "audit"; const SINGULAR_AUDIT_KIND: &str = "singular_audit"; const SQL_HOOK_KIND: &str = "sql_hook"; const CEREMONIAL_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; +const SQL_MACRO_KIND: &str = "MACRO"; const INTERPOLATION_MARKERS: [&str; 2] = ["@", "${"]; const FAULT_SEVERITY: &str = "FAULT"; const WARNING_SEVERITY: &str = "WARNING"; @@ -510,6 +516,8 @@ struct PlannedBody { input: String, sites: Vec, source: BodySource, + /// The form of a SQL macro file's body, which decides the statement lint parses it as. + macro_form: Option, } /// How a file's bodies expand, shared by each of its bodies. @@ -542,6 +550,8 @@ struct PendingBody { expanded: String, passes: Vec>, sites: Vec, + macro_form: Option, + expression_frame: Option<(i64, i64)>, } /// One file's expanded bodies and the expansion failure that stopped it, if any. @@ -731,7 +741,8 @@ fn plan_bodies( let dynamic_star: bool = !request.dynamic_output_paths.is_empty() && request.dynamic_output_paths.contains(&resolved(&file.path)); let external: Vec = external_identifiers(text, headers, &NativeHeaderParser)?; - let ceremonial: bool = headers + let sql_macro: bool = headers.iter().any(|header| header.kind == SQL_MACRO_KIND); + let mut ceremonial: bool = headers .iter() .any(|header| CEREMONIAL_KINDS.contains(&header.kind.as_str())); let mut ranges: Vec<(usize, usize)> = Vec::new(); @@ -741,15 +752,26 @@ fn plan_bodies( if ranges.is_empty() { ranges = sql_body_ranges(text, headers); } - let neutralize: Option Neutralized> = neutralizer(&BodyRole { - kind, - in_project: true, - in_hook_directory, - }); - let bodies: Vec = ranges + let neutralize: Option Neutralized> = if sql_macro { + Some(neutralize_sql_macro_parameters) + } else { + neutralizer(&BodyRole { + kind, + in_project: true, + in_hook_directory, + }) + }; + let mut bodies: Vec = ranges .into_iter() .map(|range| plan_body(text, range, neutralize, compiled)) .collect(); + if sql_macro { + for body in &mut bodies { + let form: SqlMacroForm = sql_macro_body_form(&body.input); + ceremonial = ceremonial || form == SqlMacroForm::CteList; + body.macro_form = Some(form); + } + } Ok(( BodyTraits { external_identifiers: external, @@ -835,6 +857,8 @@ pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResu expanded, passes, sites: planned.sites, + macro_form: None, + expression_frame: None, }; let prior: Vec = prior_sites(&pending.sites); let prepared: Option = py @@ -901,6 +925,7 @@ fn plan_body( input, sites, source, + macro_form: None, } } @@ -930,8 +955,8 @@ fn expand_bodies( let mut bodies: Vec = Vec::with_capacity(planned.len()); let mut failure: Option = None; let mut path: Option> = None; - for body in planned { - let (expanded, passes) = match body.source { + for mut body in planned { + let (expanded, passes) = match std::mem::replace(&mut body.source, BodySource::Plain) { BodySource::Compiled(expanded, passes) => (expanded, passes), BodySource::Plain => (body.input.clone(), Vec::new()), BodySource::Expand => { @@ -957,13 +982,7 @@ fn expand_bodies( } } }; - bodies.push(PendingBody { - start: body.start, - input: body.input, - expanded, - passes, - sites: body.sites, - }); + bodies.push(framed_body(body, (expanded, passes))); } Ok(ExpandedFile { file: index, @@ -973,6 +992,29 @@ fn expand_bodies( }) } +/// One expanded body, an expression macro body framed as the `SELECT` lint parses it as. +fn framed_body( + body: PlannedBody, + (expanded, mut passes): (String, Vec>), +) -> PendingBody { + let (expanded, expression_frame) = if body.macro_form == Some(SqlMacroForm::Expression) { + let framed: FramedExpression = frame_sql_macro_expression(&expanded); + passes.push(framed.pass); + (framed.text, Some(framed.frame)) + } else { + (expanded, None) + }; + PendingBody { + start: body.start, + input: body.input, + expanded, + passes, + sites: body.sites, + macro_form: body.macro_form, + expression_frame, + } +} + /// A compile-input error while expanding as `ProjectCompileError`; any other error unchanged. fn expansion_error(file_path: &Bound<'_, PyAny>, error: PyErr) -> PyErr { let py: Python<'_> = file_path.py(); @@ -1063,13 +1105,20 @@ fn prepared_bodies( fn lint_body( (file, traits): (usize, &BodyTraits), body: PendingBody, - (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, + (lint_text, prepared_sites, mut externally_referenced_ctes): PreparedSql, ) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); passes.push(sentinel_spans(&body.sites)); passes.extend(body.passes); passes.push(sentinel_spans(&prepared_sites)); + if body.macro_form == Some(SqlMacroForm::CteList) { + for name in defined_cte_names(&lint_text) { + if !externally_referenced_ctes.contains(&name) { + externally_referenced_ctes.push(name); + } + } + } let body = LintSqlBody { file, body_start: offset(body.start), @@ -1083,6 +1132,7 @@ fn lint_body( allows_empty_fixture_star: traits.ceremonial, relation_keys: Vec::new(), relation_names: relation_names(&dependencies.relations), + expression_frame: body.expression_frame, }; Ok((body, dependencies.relations)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 599e08ee1b..1ac55fc054 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -81,6 +81,7 @@ fn lint_body( allows_empty_fixture_star: native_field(body, "allows_empty_fixture_star")?.extract()?, relation_keys: relation_keys(body, declared)?, relation_names: relation_names(body)?, + expression_frame: None, }) } diff --git a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs index 97b85cb28c..9cbc71cc21 100644 --- a/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs +++ b/crates/sqlbuild-render/src/macro_calls/_helpers/scanner.rs @@ -3,6 +3,7 @@ use sqlbuild_core::text::main::is_letter::is_letter; use crate::macro_calls::errors::ScanError; +use crate::macro_calls::main::opens_bare_call::opens_bare_call; /// The prefix of a Python macro call after its `@`. const PYTHON_PREFIX: &str = "py."; @@ -89,11 +90,13 @@ pub(crate) fn call_start_at(text: &str, at: usize) -> CallStart { return CallStart::Sql; } if name_call_at(text, at + 1) { - let name: &str = &text[at + 1..identifier_end(text, at + 1)]; - if DECLARATION_NAMES.contains(&name) { + let name_end: usize = identifier_end(text, at + 1); + if DECLARATION_NAMES.contains(&&text[at + 1..name_end]) { return CallStart::Declaration; } - return CallStart::Bare; + if opens_bare_call(text, name_end) { + return CallStart::Bare; + } } CallStart::None } diff --git a/crates/sqlbuild-render/src/macro_calls/main/mod.rs b/crates/sqlbuild-render/src/macro_calls/main/mod.rs index 940094311c..6c6a655eb1 100644 --- a/crates/sqlbuild-render/src/macro_calls/main/mod.rs +++ b/crates/sqlbuild-render/src/macro_calls/main/mod.rs @@ -1,4 +1,5 @@ pub mod call_class_store_text; pub mod macro_call_names; +pub mod opens_bare_call; pub mod scan_macro_call_sites; pub mod splice_macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/main/opens_bare_call.rs b/crates/sqlbuild-render/src/macro_calls/main/opens_bare_call.rs new file mode 100644 index 0000000000..3cfac53d35 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/opens_bare_call.rs @@ -0,0 +1,7 @@ +//! Whether an unprefixed `@name` is a macro call, the one rule every scanner applies. + +/// Whether the `@name` ending at byte `name_end` of `text` opens a call, `(` right after it. +#[must_use] +pub fn opens_bare_call(text: &str, name_end: usize) -> bool { + text.as_bytes().get(name_end) == Some(&b'(') +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs index d685301382..35f1d718bc 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_scan_macro_call_sites.rs @@ -139,6 +139,18 @@ fn given_authored_sql_when_scanning_macro_calls_then_sites_match_python_expansio expected_sites: &[(0, 8, "a", Some(&["a"]), false)], expected_failure: Some((Some(9), ScanError::BareCall)), }, + ScanMacroCallSitesTestCase { + description: "a name followed by words and a parenthesis is not a call", + sql: "@value IS NULL OR TRIM(@value) = '' AND @x IN ('a', 'b') AND @y IS NULL AND (@y IS NULL)", + expected_sites: &[], + expected_failure: None, + }, + ScanMacroCallSitesTestCase { + description: "a name without a kind spaced from its parenthesis is not a call", + sql: "@ab (1) @py.m()", + expected_sites: &[(8, 15, "m", Some(&["m"]), false)], + expected_failure: None, + }, ScanMacroCallSitesTestCase { description: "a SQL macro call fails the Python macro scan", sql: "@sql.a(1)", diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 8f046f1c6a..b2cd6af675 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -15,6 +15,8 @@ use crate::lint_text::constants::{ }; use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; +use sqlbuild_analysis::sql_macros::constants::SQL_CALL_PREFIX; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; static PARSE_ERROR_POSITION: LazyLock> = LazyLock::new(|| Regex::new(PARSE_ERROR_POSITION_PATTERN)); @@ -133,8 +135,89 @@ fn integer(value: Option<&Value>) -> Option { } } +/// The body's lint text with each comment a SQL macro expansion generated blanked. +pub(crate) fn without_generated_comments(body: &LintSqlBody, file: &AuthoredFile<'_>) -> String { + let text: &str = &body.lint_text; + if body.passes.iter().all(Vec::is_empty) || !(text.contains("--") || text.contains("/*")) { + return text.to_owned(); + } + let bytes: &[u8] = text.as_bytes(); + let policy = lint_policy(backtick_identifiers(file.dialect)); + let mut generated: Vec<(usize, usize)> = Vec::new(); + let (mut index, mut counted_to, mut characters): (usize, usize, i64) = (0, 0, 0); + while index < bytes.len() { + match non_code_end(bytes, index, policy) { + Ok(Some(end)) => { + if matches!(bytes[index], b'-' | b'/') { + characters += text[counted_to..index].chars().count() as i64; + counted_to = index; + if from_sql_macro(body, file, characters) { + generated.push((index, end)); + } + } + index = end; + } + Ok(None) => index += 1, + Err(_) => break, + } + } + if generated.is_empty() { + return text.to_owned(); + } + let mut blanked: String = String::with_capacity(text.len()); + let mut copied_to: usize = 0; + for (start, end) in generated { + blanked.push_str(&text[copied_to..start]); + blanked.extend(text[start..end].chars().map(|character| match character { + '\n' | '\r' => character, + _ => ' ', + })); + copied_to = end; + } + blanked.push_str(&text[copied_to..]); + blanked +} + +/// The diagnostics outside a `SELECT 1` or expression frame lint added around the body. +pub(crate) fn without_generated_frame( + diagnostics: Vec, + body: &LintSqlBody, + omitted: Option, +) -> Vec { + let diagnostics: Vec = match omitted { + Some(offset) => authored_prefix(diagnostics, offset as i64), + None => diagnostics, + }; + match body.expression_frame { + Some(frame) => inside_frame(diagnostics, frame, body.lint_text.chars().count() as i64), + None => diagnostics, + } +} + +/// Whether lint text offset `offset` lies in SQL a `@sql.` call generated in the authored file. +fn from_sql_macro(body: &LintSqlBody, file: &AuthoredFile<'_>, offset: i64) -> bool { + let (mapped, generated) = map_offset(offset, &body.passes); + generated + && file + .token_at(body.body_start + mapped) + .is_some_and(|token| token.starts_with(SQL_CALL_PREFIX)) +} + +/// Keep diagnostics starting inside a framed expression, whose frame lint generated. +fn inside_frame(diagnostics: Vec, (prefix, suffix): (i64, i64), length: i64) -> Vec { + let end_of_expression: i64 = length - suffix; + let kept: Vec = diagnostics + .into_iter() + .filter(|raw| { + integer(raw.get("start")) + .is_none_or(|start| start >= prefix && start < end_of_expression) + }) + .collect(); + authored_prefix(kept, end_of_expression) +} + /// Keep diagnostics before an added `SELECT 1`, clamped to end there. -pub(crate) fn authored_prefix(diagnostics: Vec, offset: i64) -> Vec { +fn authored_prefix(diagnostics: Vec, offset: i64) -> Vec { let mut kept: Vec = Vec::new(); for raw in diagnostics { let Value::Object(mut diagnostic) = raw else { diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index 9f61cde2ae..5e487b67b3 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -5,6 +5,7 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::lint_text::_helpers::sql_diagnostics::{AuthoredFile, without_generated_comments}; use crate::lint_text::constants::{ CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, @@ -40,18 +41,19 @@ fn strings(values: &[String]) -> Value { /// The engine request for one body, with the omitted `SELECT 1` added back. pub(crate) fn lint_payload( - body: &LintSqlBody, + (body, file): (&LintSqlBody, &AuthoredFile<'_>), omitted: Option, options: &LintSqlOptions, ) -> Value { + let lint_text: String = without_generated_comments(body, file); let sql: String = match omitted { Some(offset) => { - let characters: Vec = body.lint_text.chars().collect(); + let characters: Vec = lint_text.chars().collect(); let head: String = characters[..offset].iter().collect(); let tail: String = characters[offset..].iter().collect(); format!("{head}{OMITTED_CEREMONIAL_SELECT_SQL}{tail}") } - None => body.lint_text.clone(), + None => lint_text, }; let mut payload: Map = Map::new(); let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index b38bf5ab38..47fc5ec436 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -5,6 +5,7 @@ pub const LINT_SENTINEL_PREFIX: &str = "__sqb_lint_"; pub(crate) const SENTINEL_SUFFIX: &str = "__"; pub(crate) const AUDIT_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_audit_parameter_"; pub(crate) const CONTEXT_SENTINEL_PREFIX: &str = "__sqlbuild_context_parameter_"; +pub(crate) const SQL_MACRO_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_macro_parameter_"; pub(crate) const CONTEXT_INTERPOLATION_TOKEN: &str = "@@CTX:"; /// Every SQLBuild intrinsic call protected while formatting. @@ -170,5 +171,7 @@ pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; /// What a SQL macro expression body is formatted after, and removed from again. pub(crate) const EXPRESSION_SELECT_PREFIX: &str = "SELECT "; +/// The output name lint gives an expression macro body, so it is not an unnamed projection. +pub(crate) const LINTED_EXPRESSION_ALIAS: &str = "__sqlbuild_macro_expression__"; /// The characters that open a quoted string or identifier in formatted SQL. pub(crate) const STRING_QUOTES: [char; 3] = ['\'', '"', '`']; diff --git a/crates/sqlbuild-rules/src/lint_text/main/frame_sql_macro_expression.rs b/crates/sqlbuild-rules/src/lint_text/main/frame_sql_macro_expression.rs new file mode 100644 index 0000000000..1e465ac12d --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/frame_sql_macro_expression.rs @@ -0,0 +1,39 @@ +//! `frame_sql_macro_expression`: an expression macro body as the one column of a `SELECT`. + +use crate::lint_text::constants::{EXPRESSION_SELECT_PREFIX, LINTED_EXPRESSION_ALIAS}; +use crate::lint_text::models::{ExpansionSpan, FramedExpression}; + +/// `SELECT AS `, the statement the native parser reads an expression body as. +pub fn frame_sql_macro_expression(expanded: &str) -> FramedExpression { + let mut core_end: usize = expanded.trim_end().len(); + if expanded[..core_end].ends_with(';') { + core_end = expanded[..core_end - 1].trim_end().len(); + } + let suffix: String = format!("\nAS {LINTED_EXPRESSION_ALIAS}"); + let text: String = format!( + "{EXPRESSION_SELECT_PREFIX}{}{suffix}{}", + &expanded[..core_end], + &expanded[core_end..] + ); + let prefix_chars: usize = EXPRESSION_SELECT_PREFIX.chars().count(); + let core_chars: usize = expanded[..core_end].chars().count(); + let suffix_chars: usize = suffix.chars().count(); + let offset = |value: usize| i64::try_from(value).unwrap_or(i64::MAX); + let span = |at: usize, output_start: usize, length: usize| ExpansionSpan { + source_start: offset(at), + source_end: offset(at), + output_start: offset(output_start), + output_end: offset(output_start + length), + }; + FramedExpression { + text, + pass: vec![ + span(0, 0, prefix_chars), + span(core_chars, prefix_chars + core_chars, suffix_chars), + ], + frame: ( + offset(prefix_chars), + offset(suffix_chars + expanded[core_end..].chars().count()), + ), + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs index 70f8360419..ba05dffa8e 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use serde_json::Value; use crate::lint_text::_helpers::sql_diagnostics::{ - AuthoredFile, authored_diagnostic, authored_prefix, parse_failure, response_diagnostics, + AuthoredFile, authored_diagnostic, parse_failure, response_diagnostics, without_generated_frame, }; use crate::lint_text::_helpers::sql_requests::{lint_payload, omitted_select, run_batch}; use crate::lint_text::models::{AuthoredDiagnostic, LintSqlBody, LintSqlOptions}; @@ -17,11 +17,15 @@ pub fn lint_sql_bodies( options: &LintSqlOptions, ) -> Vec { let omitted: Vec> = bodies.iter().map(omitted_select).collect(); + let authored: Vec> = files + .iter() + .map(|text| AuthoredFile::new(text, &options.dialect)) + .collect(); let mut payloads: Vec = Vec::new(); let mut index_by_request: HashMap = HashMap::new(); let mut request_of_body: Vec = Vec::with_capacity(bodies.len()); for (body, offset) in bodies.iter().zip(&omitted) { - let payload: Value = lint_payload(body, *offset, options); + let payload: Value = lint_payload((body, &authored[body.file]), *offset, options); let key: String = payload.to_string(); let next: usize = payloads.len(); let index: usize = *index_by_request.entry(key).or_insert(next); @@ -33,10 +37,6 @@ pub fn lint_sql_bodies( let responses: Vec, String>> = run_batch(&payloads) .map(|responses| responses.iter().map(response_diagnostics).collect()) .unwrap_or_else(|error| vec![Err(error); payloads.len()]); - let authored: Vec> = files - .iter() - .map(|text| AuthoredFile::new(text, &options.dialect)) - .collect(); let mut diagnostics: Vec = Vec::new(); for ((body, offset), request) in bodies.iter().zip(omitted).zip(request_of_body) { let file: &AuthoredFile<'_> = &authored[body.file]; @@ -47,10 +47,7 @@ pub fn lint_sql_bodies( continue; } }; - let raw: Vec = match offset { - Some(offset) => authored_prefix(raw, offset as i64), - None => raw, - }; + let raw: Vec = without_generated_frame(raw, body, offset); for item in &raw { match authored_diagnostic(item, body, file) { Ok(Some(diagnostic)) => diagnostics.push(diagnostic), diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 90660de7f3..38b4423133 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -2,11 +2,13 @@ pub mod dependency_sites; pub mod first_header_only; pub mod format_fixture_blocks; pub mod format_sql_bodies; +pub mod frame_sql_macro_expression; pub mod lint_sql_bodies; pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; pub mod neutralize_interpolation; +pub mod neutralize_sql_macro_parameters; pub mod reject_inline_suppressions; pub mod resolve_format_settings; pub mod restore_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_sql_macro_parameters.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_sql_macro_parameters.rs new file mode 100644 index 0000000000..cc1727b297 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_sql_macro_parameters.rs @@ -0,0 +1,23 @@ +//! `neutralize_sql_macro_parameters`: a SQL macro body's `@name` parameters replaced by sentinels. + +use sqlbuild_analysis::sql_macros::main::sql_macro_parameter_spans::sql_macro_parameter_spans; + +use crate::lint_text::_helpers::sites::{ByteSite, assemble, sentinel}; +use crate::lint_text::constants::SQL_MACRO_PARAMETER_SENTINEL_PREFIX; +use crate::lint_text::models::Neutralized; + +/// Replace each parameter reference expansion would replace, and nothing else, with a sentinel. +pub fn neutralize_sql_macro_parameters(body: &str) -> Neutralized { + let byte_sites: Vec = sql_macro_parameter_spans(body) + .into_iter() + .enumerate() + .map(|(index, (start, end))| { + ( + start, + end, + sentinel(SQL_MACRO_PARAMETER_SENTINEL_PREFIX, index), + ) + }) + .collect(); + assemble(body, byte_sites) +} diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index 7abcc7bb33..6d4ddb5c81 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -119,6 +119,17 @@ pub struct LintSqlBody { pub relation_keys: Vec<(String, Vec>)>, /// The relation name each dependency sentinel stands for. pub relation_names: Vec<(String, String)>, + /// The generated characters before and after an expression macro body framed as a `SELECT`. + pub expression_frame: Option<(i64, i64)>, +} + +/// An expression macro body framed as a `SELECT`, the pass back to the body, and the frame size. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FramedExpression { + pub text: String, + pub pass: Vec, + /// The generated characters before the expression and after it, to the end of `text`. + pub frame: (i64, i64), } /// The rule selection and thresholds of one SQL lint run. diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index 56244e605d..0b16c25b2e 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -4,10 +4,12 @@ use crate::lint_text::_helpers::sql_diagnostics::interpolation_text_at; use crate::lint_text::constants::SITE_START_BYTES; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::format_sql_bodies::format_sql_bodies; +use crate::lint_text::main::frame_sql_macro_expression::frame_sql_macro_expression; use crate::lint_text::main::lint_sql_bodies::lint_sql_bodies; use crate::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use crate::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::main::neutralize_sql_macro_parameters::neutralize_sql_macro_parameters; use crate::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use crate::lint_text::main::restore_interpolation::restore_interpolation; use crate::lint_text::models::{ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions}; @@ -138,6 +140,24 @@ fn given_generated_parameters_when_neutralizing_then_only_bare_parameters_are_re neutralize: neutralize_audit_parameters, expected_text: "select * from __sqlbuild_audit_parameter_0__ where __sqlbuild_audit_parameter_1__ < 0 and '__sqlbuild_audit_parameter_2__' = @py.f(1) and @@x", }, + ParameterTestCase { + description: "SQL macro parameters followed by words and a parenthesis", + body: "@value IS NULL OR TRIM(@value) = ''", + neutralize: neutralize_sql_macro_parameters, + expected_text: "__sqlbuild_macro_parameter_0__ IS NULL OR TRIM(__sqlbuild_macro_parameter_1__) = ''", + }, + ParameterTestCase { + description: "SQL macro parameters before IN and a parenthesised predicate", + body: "CASE WHEN @x IN ('a', 'b') THEN 1 END + CASE WHEN @x IS NULL AND (@y IS NULL) THEN 1 END", + neutralize: neutralize_sql_macro_parameters, + expected_text: "CASE WHEN __sqlbuild_macro_parameter_0__ IN ('a', 'b') THEN 1 END + CASE WHEN __sqlbuild_macro_parameter_1__ IS NULL AND (__sqlbuild_macro_parameter_2__ IS NULL) THEN 1 END", + }, + ParameterTestCase { + description: "SQL macro calls, enum references and quoted text keep their at-signs", + body: "@sql.other(@v) + @enum(\"s\").A + LENGTH('@v') + @@x", + neutralize: neutralize_sql_macro_parameters, + expected_text: "@sql.other(__sqlbuild_macro_parameter_0__) + @enum(\"s\").A + LENGTH('@v') + @@x", + }, ParameterTestCase { description: "context interpolation in hooks", body: "analyze @@CTX:relation; select @@CTX:a.b", @@ -482,3 +502,116 @@ fn given_a_formatted_expression_select_when_unwrapping_then_strings_keep_their_l ); } } + +#[test] +fn given_sql_macro_bodies_when_linting_then_frames_and_generated_comments_are_not_judged() { + let parenthesised = frame_sql_macro_expression("((x) IS NULL AND (x) IS NULL) OR TRIM(x) = ''"); + let null_comparison = frame_sql_macro_expression("x = NULL OR y;\n"); + let test_cases = [ + LintBodiesTestCase { + description: "an expression body opening with two parentheses parses as a SELECT", + contents: "((x) IS NULL AND (x) IS NULL) OR TRIM(x) = ''\n", + body: LintSqlBody { + lint_text: parenthesised.text, + passes: vec![parenthesised.pass], + expression_frame: Some(parenthesised.frame), + ..LintSqlBody::default() + }, + expected_findings: vec![], + }, + LintBodiesTestCase { + description: "a finding in a framed expression maps to the body, none to the frame", + contents: "x = NULL OR y;\n", + body: LintSqlBody { + lint_text: null_comparison.text, + passes: vec![null_comparison.pass], + expression_frame: Some(null_comparison.frame), + ..LintSqlBody::default() + }, + expected_findings: vec![("SQBRSQL001", 1, 3, Some((1, 4)))], + }, + LintBodiesTestCase { + description: "comments a macro expansion generated are not judged in the caller", + contents: "SELECT a, NOT @sql.m(b) AS c FROM t\n", + body: LintSqlBody { + lint_text: "SELECT a, NOT -- why\nCONTAINS(b, /* sign */ '@') AS c FROM t" + .to_string(), + passes: vec![vec![ExpansionSpan { + source_start: 14, + source_end: 23, + output_start: 14, + output_end: 50, + }]], + ..LintSqlBody::default() + }, + expected_findings: vec![("SQBRSQL034", 1, 1, Some((1, 7)))], + }, + LintBodiesTestCase { + description: "comments a Python macro generated are judged in the caller", + contents: "SELECT a, NOT @py.m(b) AS c FROM t\n", + body: LintSqlBody { + lint_text: "SELECT a, NOT -- why\nCONTAINS(b, /* sign */ '@') AS c FROM t" + .to_string(), + passes: vec![vec![ExpansionSpan { + source_start: 14, + source_end: 23, + output_start: 14, + output_end: 50, + }]], + ..LintSqlBody::default() + }, + expected_findings: vec![ + ("SQBRSQL034", 1, 1, Some((1, 7))), + ("SQBRSQL033", 1, 11, Some((1, 14))), + ("SQBRSQL033", 1, 15, Some((1, 23))), + ], + }, + LintBodiesTestCase { + description: "the same comments authored in the caller are judged", + contents: "SELECT a, NOT -- why\nCONTAINS(b, /* sign */ '@') AS c FROM t\n", + body: LintSqlBody { + lint_text: "SELECT a, NOT -- why\nCONTAINS(b, /* sign */ '@') AS c FROM t" + .to_string(), + ..LintSqlBody::default() + }, + expected_findings: vec![ + ("SQBRSQL034", 1, 1, Some((1, 7))), + ("SQBRSQL033", 1, 11, Some((1, 14))), + ("SQBRSQL033", 2, 11, Some((2, 12))), + ], + }, + ]; + + for test_case in test_cases { + let options = LintSqlOptions { + dialect: "duckdb".to_string(), + enabled_rules: Some( + ["SQBRSQL001", "SQBRSQL022", "SQBRSQL033", "SQBRSQL034"] + .map(str::to_string) + .to_vec(), + ), + max_literal_length: 100, + max_ranking_order_by: 5, + ..LintSqlOptions::default() + }; + + let result = lint_sql_bodies(&[test_case.body], &[test_case.contents], &options); + let findings: Vec> = result + .iter() + .map(|finding| { + ( + finding.code.as_str(), + finding.line, + finding.column, + finding.end, + ) + }) + .collect(); + + assert_eq!( + findings, test_case.expected_findings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index 4ef2ad23dc..227cc69725 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -71,6 +71,21 @@ pub(crate) fn prepare( Ok(Some((neutralized.text, neutralized.sites, referenced))) } +/// Every CTE name `sql` defines, lowercased, in order and without repeats. +pub(crate) fn defined_cte_names(sql: &str) -> Vec { + let Ok(cte_pattern) = CTES.as_ref() else { + return Vec::new(); + }; + let mut names: Vec = Vec::new(); + for capture in cte_pattern.captures_iter(sql) { + let name: String = capture["name"].to_ascii_lowercase(); + if !names.contains(&name) { + names.push(name); + } + } + names +} + /// The byte offset of character position `position`, or `None` past the end. fn byte_offset(text: &str, position: usize) -> Option { text.char_indices() diff --git a/crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs b/crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs new file mode 100644 index 0000000000..c7177d64a9 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/defined_cte_names.rs @@ -0,0 +1,7 @@ +//! The CTE names a SQL text defines, as lint preparation reads them. + +/// Every CTE name `sql` defines, lowercased, in order and without repeats. +#[must_use] +pub fn defined_cte_names(sql: &str) -> Vec { + crate::sql_lint::_helpers::preparation::defined_cte_names(sql) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs index fdc751309d..7fbbc4fae5 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -2,6 +2,7 @@ pub mod backtick_identifiers; pub mod batch_engine; pub mod catalogue; pub mod cross_join_subject; +pub mod defined_cte_names; pub mod engine; pub mod fill_finding_path; pub mod format_requests; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py index 9f68f5e5c4..147ec3b2f3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/_test_types.py @@ -68,3 +68,15 @@ class SqlMacroRecompileTestCase: replaced_text: tuple[str, str] expected_code: str expected_fragment: str + + +@dataclass(frozen=True) +class SqlMacroCompileTestCase: + """One project compiled, what its output and inspected files hold, and what they must not.""" + + description: str + files: dict[str, str] + expected_returncode: int + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] = () + inspected_paths: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py index f48a24787b..19e9278d05 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/helpers.py @@ -165,3 +165,43 @@ def inspected_text(*, project_dir: Path, relative_paths: tuple[str, ...]) -> str (project_dir / relative_path).read_text(encoding="utf-8") for relative_path in relative_paths ) + + +WRAPPING_MACROS: dict[str, str] = { + "macros/python/wrapping.py": ( + "def wrap(expr: str) -> str:\n" + ' """Wrap an expression in parentheses."""\n' + ' return f"({expr})"\n\n\n' + "def pick(mapping: dict) -> str:\n" + ' """The expression under key a."""\n' + ' return mapping["a"]\n\n\n' + "def first(items: list) -> str:\n" + ' """The first expression of a list."""\n' + " return items[0]\n\n\n" + "def noted(expr: str) -> str:\n" + ' """An expression and a block comment."""\n' + ' return f"UPPER({expr}) /* noted */"\n' + ) +} +CHECK_MACRO_PATH: str = "macros/sql/check.sql" + + +def with_rules(*codes: str) -> dict[str, str]: + """The base project with the given Rules selected.""" + + selected: str = ", ".join(f'"{code}"' for code in codes) + return { + "sqlbuild_project.toml": BASE_FILES["sqlbuild_project.toml"] + + f"\n[rules]\nselect = [{selected}]\n" + } + + +def check_macro(arguments: str, body: str) -> dict[str, str]: + """The SQL macro `check` returning a boolean from `body`.""" + + return { + CHECK_MACRO_PATH: ( + f'MACRO (\n description "A check",\n arguments ({arguments}),\n' + f" returns BOOLEAN,\n);\n\n{body}\n" + ) + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py new file mode 100644 index 0000000000..9d6c734443 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_expansion_text.py @@ -0,0 +1,430 @@ +"""E2E coverage for SQL macro bodies under Rules and SQL macro expansions passed to Python macros.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros._test_types import ( + SqlMacroCompileTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_macros.helpers import ( + CHECK_MACRO_PATH, + WRAPPING_MACROS, + check_macro, + inspected_text, + orders_model, + output, + run_project, + with_rules, +) + +_COMPILED: str = "target/compiled/models/marts/orders.sql" +_RULE_FAILURES: tuple[str, ...] = ("L005", "rules-unevaluated", "Native SQL parser") +_CHECK_STATUS: dict[str, str] = orders_model( + 'SELECT order_id, @sql.check(status) AS checked FROM __ref("stg_orders")\n' +) +_COMMENTED_CHECK: dict[str, str] = check_macro( + "value VARCHAR", + "-- a status holding an at sign is flagged\nCONTAINS(@value, /* the sign */ '@')", +) +_COMMAND: tuple[str, ...] = ("compile", "--no-cache") + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="parameter_before_words_and_a_parenthesis", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "@value IS NULL OR TRIM(@value) = ''"), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=("status IS NULL OR TRIM(status) = '' AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="parameter_before_in_inside_case", + files={ + **with_rules("SQBRSQL001"), + **check_macro( + "value VARCHAR", + "CASE WHEN @value IN ('open', 'closed') THEN TRUE ELSE FALSE END", + ), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=( + "CASE WHEN status IN ('open', 'closed') THEN TRUE ELSE FALSE END AS checked", + ), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="parameter_before_and_a_parenthesised_predicate", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "@value IS NULL AND (@value IS NULL)"), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=("status IS NULL AND (status IS NULL) AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="parameter_before_in_alone", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "@value IN ('', ' ')"), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=("status IN ('', ' ') AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="parameter_compared_then_trimmed", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "@value = '' OR TRIM(@value) = ''"), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=("status = '' OR TRIM(status) = '' AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="parenthesised_parameter", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "(@value) IS NULL OR TRIM(@value) = ''"), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=("(status) IS NULL OR TRIM(status) = '' AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="body_opening_with_two_parentheses", + files={ + **with_rules("SQBRSQL001"), + **check_macro( + "value VARCHAR", "((@value) IS NULL AND (@value) IS NULL) OR TRIM(@value) = ''" + ), + **_CHECK_STATUS, + }, + expected_returncode=0, + expected_fragments=( + "((status) IS NULL AND (status) IS NULL) OR TRIM(status) = '' AS checked", + ), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="spaced_enum_and_constant_references_in_a_body_are_declarations", + files={ + **with_rules("SQBRSQL001"), + "constants/fee_divisor.sql": "CONSTANT (name fee_divisor, value 100);\n", + **check_macro( + "value VARCHAR, cents BIGINT", + '@value <> @enum ("order_status").CLOSED AND @cents / @const ("fee_divisor") > 1', + ), + **orders_model( + "SELECT order_id, @sql.check(status, amount_cents) AS checked " + 'FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("status <> 'closed' AND amount_cents / 100 > 1 AS checked",), + unexpected_fragments=(*_RULE_FAILURES, "P022"), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="a_trailing_comment_before_a_line_break_compiles_unchanged", + files={ + **check_macro("value VARCHAR", "CONTAINS(@value, 'x') -- why"), + **orders_model('SELECT order_id, @sql.check(status)\nFROM __ref("stg_orders")\n'), + }, + expected_returncode=0, + expected_fragments=("SELECT order_id, CONTAINS(status, 'x') -- why\nFROM __ref(",), + unexpected_fragments=("-- why\n\n",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="two_parameters_before_and_a_parenthesised_predicate", + files={ + **with_rules("SQBRSQL001"), + **check_macro("x VARCHAR, y VARCHAR", "@x IS NULL AND (@y IS NULL)"), + **orders_model( + "SELECT order_id, @sql.check(status, country_code) AS checked " + 'FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("status IS NULL AND (country_code IS NULL) AS checked",), + unexpected_fragments=_RULE_FAILURES, + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="a_finding_in_a_macro_body_is_reported_in_the_macro_file", + files={ + **with_rules("SQBRSQL001"), + **check_macro("value VARCHAR", "((@value) IS NULL) OR @value = NULL"), + **_CHECK_STATUS, + }, + expected_returncode=1, + expected_fragments=("[SQBRSQL001]", f"--> {CHECK_MACRO_PATH}:7:"), + unexpected_fragments=_RULE_FAILURES, + ), + SqlMacroCompileTestCase( + description="a_cte_list_body_lints_without_parse_or_unused_cte_findings", + files={ + **with_rules("SQBRSQL001", "SQBRSQL005"), + "macros/sql/live_orders.sql": ( + 'MACRO (\n description "Live orders",\n' + " arguments (src RELATION (order_id BIGINT, is_deleted BOOLEAN)),\n);\n\n" + "WITH live AS (SELECT s.order_id FROM @src AS s WHERE NOT s.is_deleted),\n" + "live_ids AS (SELECT live.order_id FROM live)\n" + ), + **orders_model( + 'WITH raw_orders AS (SELECT * FROM __ref("stg_orders")),\n' + "@sql.live_orders(raw_orders)\nSELECT order_id FROM live_ids\n" + ), + }, + expected_returncode=0, + expected_fragments=("Project compiled",), + unexpected_fragments=(*_RULE_FAILURES, "SQBRSQL005"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_sql_macro_body_when_compiling_with_rules_then_it_lints_and_expands( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project(tmp_path=tmp_path, files=test_case.files, command=_COMMAND) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="double_quote_in_a_string_literal", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "CONTAINS(@value, '\"')"), + **orders_model( + 'SELECT order_id, @py.wrap(@sql.check(status)) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("(CONTAINS(status, '\"')) AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="double_quote_compared_directly", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "@value = '\"'"), + **orders_model( + 'SELECT order_id, @py.wrap(@sql.check(status)) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("(status = '\"') AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="double_quote_as_a_dict_value", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "CONTAINS(@value, '\"')"), + **orders_model( + 'SELECT order_id, @py.pick({"a": @sql.check(status)}) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("CONTAINS(status, '\"') AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="quotes_and_backslashes_in_a_nested_list", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "@value IN ('it''s', '\\\\', '\"')"), + **orders_model( + 'SELECT order_id, @py.first([@sql.check(status), [@sql.check(status)]]) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("status IN ('it''s', '\\\\', '\"') AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="line_comment_before_the_expression", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "-- why\nCONTAINS(@value, 'x')"), + **orders_model( + 'SELECT order_id, @py.wrap(@sql.check(status)) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("(-- why\nCONTAINS(status, 'x')) AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="block_comment_inside_the_expression", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "CONTAINS(@value, /* why */ 'x')"), + **orders_model( + 'SELECT order_id, @py.wrap(@sql.check(status)) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("(CONTAINS(status, /* why */ 'x')) AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="line_comment_ending_the_body", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "CONTAINS(@value, 'x') -- why"), + **orders_model( + 'SELECT order_id, @py.wrap(@sql.check(status)) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("(CONTAINS(status, 'x') -- why\n) AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="line_comment_ending_a_body_called_directly", + files={ + **WRAPPING_MACROS, + **check_macro("value VARCHAR", "CONTAINS(@value, 'x') -- why"), + **orders_model( + 'SELECT order_id, @sql.check(status) AS probe FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=0, + expected_fragments=("CONTAINS(status, 'x') -- why\n AS probe",), + unexpected_fragments=("P001",), + inspected_paths=(_COMPILED,), + ), + SqlMacroCompileTestCase( + description="an_unreadable_python_macro_call_names_its_place", + files={ + **WRAPPING_MACROS, + **orders_model( + 'SELECT order_id,\n @py.wrap(\'oops) AS probe\nFROM __ref("stg_orders")\n' + ), + }, + expected_returncode=1, + expected_fragments=( + "Macro arguments of '@py.wrap' in '", + "orders.sql' could not be read: a quoted string in @py.wrap is not closed " + "at line 4, column 3", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_sql_macro_call_in_python_macro_arguments_when_compiling_then_python_gets_its_sql( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + project_dir, result = run_project(tmp_path=tmp_path, files=test_case.files, command=_COMMAND) + printed: str = output(result) + inspected_text( + project_dir=project_dir, relative_paths=test_case.inspected_paths + ) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed + + +@pytest.mark.parametrize( + "test_case", + [ + SqlMacroCompileTestCase( + description="comments_from_a_macro_expansion_are_not_judged_in_the_caller", + files={ + **with_rules("SQBRSQL033"), + **_COMMENTED_CHECK, + **orders_model( + "SELECT order_id, NOT @sql.check(status) AS plain, " + 'COALESCE(@sql.check(status), FALSE) AS checked FROM __ref("stg_orders")\n' + ), + }, + expected_returncode=1, + expected_fragments=(f"--> {CHECK_MACRO_PATH}:",), + unexpected_fragments=("--> models/marts/orders.sql",), + ), + SqlMacroCompileTestCase( + description="comments_a_python_macro_returns_are_judged_in_the_caller", + files={ + **with_rules("SQBRSQL033"), + **WRAPPING_MACROS, + **orders_model( + "SELECT order_id, @py.noted('status') AS noted FROM __ref(\"stg_orders\")\n" + ), + }, + expected_returncode=1, + expected_fragments=( + "[SQBRSQL033]", + "(in SQL generated by @py.noted('status'))", + "--> models/marts/orders.sql:3:", + ), + ), + SqlMacroCompileTestCase( + description="the_same_comments_written_in_a_model_are_judged", + files={ + **with_rules("SQBRSQL033"), + **orders_model( + "SELECT order_id, NOT -- a status holding an at sign is flagged\n" + "CONTAINS(status, /* the sign */ '@') AS plain FROM __ref(\"stg_orders\")\n" + ), + }, + expected_returncode=1, + expected_fragments=("[SQBRSQL033]", "--> models/marts/orders.sql:3:"), + ), + ], + ids=lambda case: case.description, +) +def test_given_comments_in_a_sql_macro_when_linting_its_caller_then_only_authored_comments_count( + test_case: SqlMacroCompileTestCase, tmp_path: Path +) -> None: + _, result = run_project(tmp_path=tmp_path, files=test_case.files, command=_COMMAND) + printed: str = output(result) + + assert result.returncode == test_case.expected_returncode, printed + for fragment in test_case.expected_fragments: + assert fragment in printed, printed + for fragment in test_case.unexpected_fragments: + assert fragment not in printed, printed diff --git a/tests/goldens/python_oracle/macro_expansion_nested_argument_scan_edges.json b/tests/goldens/python_oracle/macro_expansion_nested_argument_scan_edges.json index 34b9d298c4..b8b1af7b18 100644 --- a/tests/goldens/python_oracle/macro_expansion_nested_argument_scan_edges.json +++ b/tests/goldens/python_oracle/macro_expansion_nested_argument_scan_edges.json @@ -5,7 +5,7 @@ ], [ "macro_expansion", -"ff90ea8ba463db2d44e11898" +"e5bbf689186f3c18d715ba20" ], [ "macro_expansion", @@ -13,11 +13,11 @@ ], [ "macro_expansion", -"ac936e3b930d7120425ccf2d" +"5915a019257cf7407eee15d3" ], [ "macro_expansion", -"ac936e3b930d7120425ccf2d" +"5915a019257cf7407eee15d3" ], [ "macro_expansion", @@ -33,7 +33,7 @@ ], [ "macro_expansion", -"ac936e3b930d7120425ccf2d" +"5915a019257cf7407eee15d3" ], [ "macro_expansion", @@ -41,11 +41,11 @@ ], [ "macro_expansion", -"1d9a010df82079fdb5727766" +"190febf661200a407d59746e" ], [ "macro_expansion", -"1d9a010df82079fdb5727766" +"46a35bc0512271fd917753b4" ], [ "macro_expansion", @@ -73,7 +73,7 @@ ], [ "macro_expansion", -"ea0e6e81338954cb22c3c4dc" +"3a0174e0036ca09a54d7d4ef" ], [ "macro_expansion", diff --git a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py index 3a79cbb677..b016a58815 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py +++ b/tests/integration/src/sqlbuild/compiler/compile/test_native_macro_bridge.py @@ -156,7 +156,7 @@ def test_given_repeated_macro_calls_when_rendering_natively_then_each_call_class 'SELECT @py.cents("amount") AS a, @py.cents x("amount") AS b\n' ), }, - expected_message_fragment="expected opening parenthesis", + expected_message_fragment="is not followed by its opening parenthesis", ), ], ids=lambda case: case.description, From a375f7094b904bf43f12b612862319bb74ccdd27 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:07:37 +0100 Subject: [PATCH 2/2] fix: build authored lint lookups after the lint batch so SQL macro comment blanking adds no peak memory --- .../src/lint_text/_helpers/sql_diagnostics.rs | 28 ++++++++++++------- .../src/lint_text/_helpers/sql_requests.rs | 10 ++++--- .../src/lint_text/main/lint_sql_bodies.rs | 10 +++---- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index b2cd6af675..a0c52b771b 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -1,5 +1,6 @@ //! Native SQL lint responses mapped back onto authored source through expansion passes. +use std::borrow::Cow; use std::sync::LazyLock; use regex::Regex; @@ -136,13 +137,16 @@ fn integer(value: Option<&Value>) -> Option { } /// The body's lint text with each comment a SQL macro expansion generated blanked. -pub(crate) fn without_generated_comments(body: &LintSqlBody, file: &AuthoredFile<'_>) -> String { +pub(crate) fn without_generated_comments<'b>( + body: &'b LintSqlBody, + file: (&str, &str), +) -> Cow<'b, str> { let text: &str = &body.lint_text; if body.passes.iter().all(Vec::is_empty) || !(text.contains("--") || text.contains("/*")) { - return text.to_owned(); + return Cow::Borrowed(text); } let bytes: &[u8] = text.as_bytes(); - let policy = lint_policy(backtick_identifiers(file.dialect)); + let policy = lint_policy(backtick_identifiers(file.1)); let mut generated: Vec<(usize, usize)> = Vec::new(); let (mut index, mut counted_to, mut characters): (usize, usize, i64) = (0, 0, 0); while index < bytes.len() { @@ -162,7 +166,7 @@ pub(crate) fn without_generated_comments(body: &LintSqlBody, file: &AuthoredFile } } if generated.is_empty() { - return text.to_owned(); + return Cow::Borrowed(text); } let mut blanked: String = String::with_capacity(text.len()); let mut copied_to: usize = 0; @@ -175,7 +179,7 @@ pub(crate) fn without_generated_comments(body: &LintSqlBody, file: &AuthoredFile copied_to = end; } blanked.push_str(&text[copied_to..]); - blanked + Cow::Owned(blanked) } /// The diagnostics outside a `SELECT 1` or expression frame lint added around the body. @@ -195,12 +199,16 @@ pub(crate) fn without_generated_frame( } /// Whether lint text offset `offset` lies in SQL a `@sql.` call generated in the authored file. -fn from_sql_macro(body: &LintSqlBody, file: &AuthoredFile<'_>, offset: i64) -> bool { +fn from_sql_macro(body: &LintSqlBody, (text, dialect): (&str, &str), offset: i64) -> bool { let (mapped, generated) = map_offset(offset, &body.passes); - generated - && file - .token_at(body.body_start + mapped) - .is_some_and(|token| token.starts_with(SQL_CALL_PREFIX)) + if !generated { + return false; + } + match usize::try_from(body.body_start + mapped) { + Ok(start) => interpolation_text_at(text, start, dialect) + .is_some_and(|token| token.starts_with(SQL_CALL_PREFIX)), + Err(_) => false, + } } /// Keep diagnostics starting inside a framed expression, whose frame lint generated. diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index 5e487b67b3..32f38b4e83 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -1,11 +1,13 @@ //! Native SQL lint request payloads and the batch engine call. +use std::borrow::Cow; + use serde_json::{Map, Value}; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::lint_text::_helpers::sql_diagnostics::{AuthoredFile, without_generated_comments}; +use crate::lint_text::_helpers::sql_diagnostics::without_generated_comments; use crate::lint_text::constants::{ CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, @@ -41,11 +43,11 @@ fn strings(values: &[String]) -> Value { /// The engine request for one body, with the omitted `SELECT 1` added back. pub(crate) fn lint_payload( - (body, file): (&LintSqlBody, &AuthoredFile<'_>), + (body, file): (&LintSqlBody, &str), omitted: Option, options: &LintSqlOptions, ) -> Value { - let lint_text: String = without_generated_comments(body, file); + let lint_text: Cow<'_, str> = without_generated_comments(body, (file, &options.dialect)); let sql: String = match omitted { Some(offset) => { let characters: Vec = lint_text.chars().collect(); @@ -53,7 +55,7 @@ pub(crate) fn lint_payload( let tail: String = characters[offset..].iter().collect(); format!("{head}{OMITTED_CEREMONIAL_SELECT_SQL}{tail}") } - None => lint_text, + None => lint_text.into_owned(), }; let mut payload: Map = Map::new(); let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); diff --git a/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs index ba05dffa8e..be43f45778 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs @@ -17,15 +17,11 @@ pub fn lint_sql_bodies( options: &LintSqlOptions, ) -> Vec { let omitted: Vec> = bodies.iter().map(omitted_select).collect(); - let authored: Vec> = files - .iter() - .map(|text| AuthoredFile::new(text, &options.dialect)) - .collect(); let mut payloads: Vec = Vec::new(); let mut index_by_request: HashMap = HashMap::new(); let mut request_of_body: Vec = Vec::with_capacity(bodies.len()); for (body, offset) in bodies.iter().zip(&omitted) { - let payload: Value = lint_payload((body, &authored[body.file]), *offset, options); + let payload: Value = lint_payload((body, files[body.file]), *offset, options); let key: String = payload.to_string(); let next: usize = payloads.len(); let index: usize = *index_by_request.entry(key).or_insert(next); @@ -37,6 +33,10 @@ pub fn lint_sql_bodies( let responses: Vec, String>> = run_batch(&payloads) .map(|responses| responses.iter().map(response_diagnostics).collect()) .unwrap_or_else(|error| vec![Err(error); payloads.len()]); + let authored: Vec> = files + .iter() + .map(|text| AuthoredFile::new(text, &options.dialect)) + .collect(); let mut diagnostics: Vec = Vec::new(); for ((body, offset), request) in bodies.iter().zip(omitted).zip(request_of_body) { let file: &AuthoredFile<'_> = &authored[body.file];