diff --git a/crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs index 2ff74ddcfd..76a90afcda 100644 --- a/crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules-native/src/configuration/_helpers/loading.rs @@ -99,7 +99,9 @@ fn validate_raw(value: &toml::Value) -> Result<(), String> { } pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { - const THRESHOLDS: [&str; 10] = [ + const THRESHOLDS: [&str; 12] = [ + "max_literal_length", + "max_ranking_order_by", "min_audits_per_model", "min_tests_per_model", "min_custom_rule_test_cases", @@ -160,6 +162,14 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { { return Err(format!("unknown rule thresholds: {name}")); } + if let Some(name) = ["max_literal_length", "max_ranking_order_by"] + .into_iter() + .find(|name| config.thresholds.get(*name) == Some(&0)) + { + return Err(format!( + "rules.thresholds.{name} must be a positive integer" + )); + } for entry in &config.threshold_overrides { if entry.paths.is_empty() || entry.thresholds.is_empty() || entry.reason.trim().is_empty() { return Err("rules.threshold_overrides requires paths, thresholds, and reason".into()); diff --git a/crates/sqlbuild-rules-native/src/configuration/main/mod.rs b/crates/sqlbuild-rules-native/src/configuration/main/mod.rs index f418064f12..22abc37c06 100644 --- a/crates/sqlbuild-rules-native/src/configuration/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/configuration/main/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod load; +pub(crate) mod setting_help; +pub(crate) mod toml_string_array; pub(crate) mod validate; diff --git a/crates/sqlbuild-rules-native/src/configuration/main/setting_help.rs b/crates/sqlbuild-rules-native/src/configuration/main/setting_help.rs new file mode 100644 index 0000000000..4cf8eabbf2 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/configuration/main/setting_help.rs @@ -0,0 +1,10 @@ +//! Remediation text that shows the exact `sqlbuild_project.toml` setting to write. + +use crate::constants::{PROJECT_CONFIG_FILE, SETTING_SNIPPET_INDENT}; + +/// `, set this in sqlbuild_project.toml:` followed by the section and `key = value`. +pub(crate) fn setting_help(purpose: &str, section: &str, key: &str, value: &str) -> String { + format!( + "{purpose}, set this in {PROJECT_CONFIG_FILE}:\n{SETTING_SNIPPET_INDENT}[{section}]\n{SETTING_SNIPPET_INDENT}{key} = {value}" + ) +} diff --git a/crates/sqlbuild-rules-native/src/configuration/main/toml_string_array.rs b/crates/sqlbuild-rules-native/src/configuration/main/toml_string_array.rs new file mode 100644 index 0000000000..e2c7bc909d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/configuration/main/toml_string_array.rs @@ -0,0 +1,10 @@ +//! TOML array rendering for remediation snippets. + +/// A TOML array of strings, in the given order. +pub(crate) fn toml_string_array<'a>(values: impl IntoIterator) -> String { + let quoted: Vec = values + .into_iter() + .map(|value| format!("{value:?}")) + .collect(); + format!("[{}]", quoted.join(", ")) +} diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index 528c0ea332..bbd77bbedb 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -22,6 +22,7 @@ pub(crate) const ENFORCED_CONTRACT: &str = "enforced"; pub(crate) const SELECT_STAR_MODEL_RULE_CODE: &str = "SQBRMODEL102"; pub(crate) const BOOLEAN_TYPE: &str = "BOOLEAN"; pub(crate) const TIMESTAMP_TYPE: &str = "TIMESTAMP"; +pub(crate) const DATETIME_TYPE: &str = "DATETIME"; pub(crate) const DATE_TYPE: &str = "DATE"; pub(crate) const NEGATION_OPERATOR: &str = "-"; pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; @@ -53,3 +54,5 @@ pub(crate) const SQL_TEST_ACTUAL_CTE: &str = "__actual"; pub(crate) const SQL_TEST_EXPECTED_CTE: &str = "__expected"; pub(crate) const SQL_TEST_ACTUAL_CTE_PREFIX: &str = "__actual__"; pub(crate) const QUOTED_IDENTIFIER_DELIMITER_BYTES: usize = 2; +pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; +pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; diff --git a/crates/sqlbuild-rules-native/src/lib.rs b/crates/sqlbuild-rules-native/src/lib.rs index 2b36ba76d5..10c52999f9 100644 --- a/crates/sqlbuild-rules-native/src/lib.rs +++ b/crates/sqlbuild-rules-native/src/lib.rs @@ -13,5 +13,6 @@ mod scope_metadata; mod semantic_usage; mod semantic_validation; mod sql_lint; +mod sql_quality; mod sql_scan; mod sql_tokens; diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs index 537c9898cf..4b24a49267 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs @@ -1,3 +1,4 @@ +use crate::configuration::main::setting_help::setting_help; use crate::models::{DeclarationKind, Fault, Model, RuleMetadata, ScopeDeclaration, ScopeKind}; use crate::rules::_helpers::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; @@ -187,7 +188,12 @@ fn evaluate_models(evaluation: &ProjectEvaluationRequest<'_>) -> Vec { "model has ambiguous domain-root and level candidates: {}", candidates.join(", ") ), - "Set rules.layout.domain_roots to the intended normalized roots.".into(), + setting_help( + "List the intended normalized domain roots", + "rules.layout", + "domain_roots", + r#"[""]"#, + ), )); } continue; @@ -207,10 +213,15 @@ fn evaluate_models(evaluation: &ProjectEvaluationRequest<'_>) -> Vec { max_depth ), format!( - "Flatten this path beneath {}/{}, promote part of it into the domain root, or set rules.thresholds.max_subdomain_depth to at least {}.", + "Flatten this path beneath {}/{}, or promote part of it into the domain root. {}", resolved.domain, resolved.level, - resolved.owners.len() + setting_help( + &format!("To allow this depth (the current value is {max_depth})"), + "rules.thresholds", + "max_subdomain_depth", + &resolved.owners.len().to_string(), + ) ), )); } @@ -513,12 +524,20 @@ fn inspect_container( format!( "declaration role {:?} has bucket depth {}; configured maximum is {}", inspection.root, - entry.buckets.len() - ,inspection.max_depth + entry.buckets.len(), + inspection.max_depth ), format!( - "Flatten this bucket path or set rules.thresholds.max_role_container_depth to at least {}.", - entry.buckets.len() + "Flatten this bucket path. {}", + setting_help( + &format!( + "To allow this depth (the current value is {})", + inspection.max_depth + ), + "rules.thresholds", + "max_role_container_depth", + &entry.buckets.len().to_string(), + ) ), )); } diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs index 5ee489c8fb..f4bf74d34a 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs @@ -1,3 +1,5 @@ +use crate::configuration::main::setting_help::setting_help; +use crate::configuration::main::toml_string_array::toml_string_array; use crate::constants::{ DECLARATION_DOMAIN_COMPONENTS, ENFORCED_CONTRACT, NEGATION_OPERATOR, REFERENCE_KIND, SOURCE_REFERENCE_KIND, VIEW_MATERIALIZATION, @@ -1102,7 +1104,22 @@ fn name_grammar( "model {:?} uses unknown domain {:?}", parsed.model.name, parts.domain ), - Some("Rename the model into a configured rules domain, or add this domain to rules.domains when it is an intentional project owner.".into()), + Some(format!( + "Rename the model into a configured rules domain. The current value is [rules] domains = {}. {}", + toml_string_array(config.domains.iter().map(String::as_str)), + setting_help( + "If this domain is an intentional project owner, add it", + "rules", + "domains", + &toml_string_array( + config + .domains + .iter() + .map(String::as_str) + .chain([parts.domain.as_str()]), + ), + ) + )), )); } return; @@ -1208,7 +1225,22 @@ fn source_token_rule( "model {:?} uses unapproved source token {:?}", parsed.model.name, token ), - Some("Rename the source suffix to a token listed in rules.approved_source_tokens at this model path.".into()), + Some(format!( + "Rename the source suffix to an approved token. The current value is [rules] approved_source_tokens = {}. {}", + toml_string_array(config.approved_source_tokens.iter().map(String::as_str)), + setting_help( + "To approve this token", + "rules", + "approved_source_tokens", + &toml_string_array( + config + .approved_source_tokens + .iter() + .map(String::as_str) + .chain([token.as_str()]), + ), + ) + )), )); } } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs index b4945081ab..80bb11cdff 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs @@ -2,6 +2,7 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; +use crate::semantic_validation::_helpers::type_locations::locate_type_findings; use crate::semantic_validation::models::FunctionProbes; use crate::semantic_validation::types::DiagnosticRow; use polyglot_sql::expressions::Select; @@ -153,8 +154,10 @@ pub(crate) fn map_diagnostics( dialect, statements: None, }; - let mut errors: Vec = Vec::with_capacity(result.errors.len()); - for error in result.errors { + let mut native = result.errors; + locate_type_findings(sql, dialect, &mut native)?; + let mut errors: Vec = Vec::with_capacity(native.len()); + for error in native { if let Some(error) = context.map(error)? { errors.push(error); } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs index a0cffe880f..11d6c2854e 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod function_names; pub(crate) mod identifiers; pub(crate) mod normalization; pub(crate) mod positions; +pub(crate) mod type_locations; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs new file mode 100644 index 0000000000..8a338529a2 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/type_locations.rs @@ -0,0 +1,435 @@ +//! Authored locations for native cast and set-operation findings that carry no span. + +use crate::constants::{DATETIME_TYPE, TIMESTAMP_TYPE}; +use polyglot_sql::tokens::{Token, TokenType}; + +use crate::sql_tokens::main::projection_spans::projection_spans; +use polyglot_sql::{Dialect, DialectType, ValidationError}; +use regex::Regex; +use std::sync::LazyLock; + +const INVALID_CAST: &str = "E218"; +const LOSSY_CAST: &str = "W213"; +const SET_OPERATION_CODES: [&str; 3] = ["E215", "E216", "W214"]; +static CANNOT_CAST: LazyLock> = LazyLock::new(|| { + Regex::new(r"^Cannot cast (\w+) to (\w+)$").map_err(|error| error.to_string()) +}); +static UNKNOWN_CAST_TYPE: LazyLock> = LazyLock::new(|| { + Regex::new(r"^Unknown cast target type '([^']+)'$").map_err(|error| error.to_string()) +}); +static SET_OPERATION_COLUMN: LazyLock> = LazyLock::new(|| { + Regex::new(r"^Set-operation column ([1-9][0-9]*) ").map_err(|error| error.to_string()) +}); +const COMPARISON_LITERAL: &str = "Comparison contains an invalid coercible literal"; +const IN_LIST_LITERAL: &str = "IN list contains an invalid coercible literal"; +const TEMPORAL_LITERAL: &str = "Invalid temporal literal"; +const INTERVAL_LITERAL: &str = "Invalid interval literal"; +const COMPARISON_OPERATORS: [TokenType; 7] = [ + TokenType::Eq, + TokenType::Neq, + TokenType::Lt, + TokenType::Gt, + TokenType::Lte, + TokenType::Gte, + TokenType::NullsafeEq, +]; +const SET_OPERATION_ARITY: &str = "Set-operation operands return different column counts"; + +struct CastSite { + start: usize, + end: usize, + target: String, + operand_words: Vec, + literal_operand: bool, +} + +struct SetOperationSite { + keyword: (usize, usize), + projections: Vec<(usize, usize)>, +} + +/// Point each position-less cast, literal and set-operation finding at its own expression. +pub(crate) fn locate_type_findings( + sql: &str, + dialect: DialectType, + errors: &mut [ValidationError], +) -> Result<(), String> { + if !errors.iter().any(|error| { + error.start.is_none() && (is_cast(&error.code) || is_set_operation(&error.code)) + }) { + return Ok(()); + } + let Ok(tokens) = Dialect::get(dialect).tokenize(sql) else { + return Ok(()); + }; + let positioned: Vec = errors + .iter() + .filter(|error| is_cast(&error.code)) + .filter_map(|error| error.start) + .collect(); + let mut locator = Locator::new(&tokens, &positioned); + for error in errors.iter_mut().filter(|error| error.start.is_none()) { + if let Some((start, end)) = locator.span(&error.code, &error.message)? { + error.start = Some(start); + error.end = Some(end); + } + } + Ok(()) +} + +struct Locator { + casts: Vec, + used_casts: Vec, + set_operations: Vec, + used_columns: Vec>, + used_arity: Vec, + literals: LiteralSites, +} + +impl Locator { + fn new(tokens: &[Token], positioned: &[usize]) -> Self { + let casts: Vec = cast_sites(tokens); + let used_casts: Vec = casts + .iter() + .map(|site| contains_any(site, positioned)) + .collect(); + let set_operations: Vec = set_operation_sites(tokens); + let used_columns: Vec> = set_operations + .iter() + .map(|site| vec![false; site.projections.len()]) + .collect(); + let used_arity: Vec = vec![false; set_operations.len()]; + Self { + casts, + used_casts, + set_operations, + used_columns, + used_arity, + literals: LiteralSites::new(tokens), + } + } + + fn span(&mut self, code: &str, message: &str) -> Result, String> { + if is_cast(code) { + if let Some(span) = self.literals.next(message) { + return Ok(Some(span)); + } + return self.cast_span(message); + } + if is_set_operation(code) { + return self.set_operation_span(message); + } + Ok(None) + } + + fn cast_span(&mut self, message: &str) -> Result, String> { + let Some(preference) = CastPreference::from_message(message)? else { + return Ok(None); + }; + let index: Option = (0..self.casts.len()) + .filter(|&index| !self.used_casts[index]) + .min_by_key(|&index| (preference.rank(&self.casts[index]), index)); + Ok(index.map(|index| { + self.used_casts[index] = true; + (self.casts[index].start, self.casts[index].end) + })) + } + + fn set_operation_span(&mut self, message: &str) -> Result, String> { + if message.starts_with(SET_OPERATION_ARITY) { + let index: Option = + (0..self.set_operations.len()).find(|&index| !self.used_arity[index]); + return Ok(index.map(|index| { + self.used_arity[index] = true; + self.set_operations[index].keyword + })); + } + let Some(captures) = SET_OPERATION_COLUMN + .as_ref() + .map_err(Clone::clone)? + .captures(message) + else { + return Ok(None); + }; + let Ok(ordinal) = captures[1].parse::() else { + return Ok(None); + }; + let column: usize = ordinal - 1; + let index: Option = (0..self.set_operations.len()).find(|&index| { + self.set_operations[index].projections.len() > column + && !self.used_columns[index][column] + }); + Ok(index.map(|index| { + self.used_columns[index][column] = true; + self.set_operations[index].projections[column] + })) + } +} + +fn contains_any(site: &CastSite, starts: &[usize]) -> bool { + starts + .iter() + .any(|&start| site.start <= start && start < site.end) +} + +/// How well a cast site explains one cast finding; lower ranks are better matches. +enum CastPreference { + Conversion { source: String, target: String }, + UnknownTarget(String), +} + +impl CastPreference { + fn from_message(message: &str) -> Result, String> { + if let Some(captures) = CANNOT_CAST + .as_ref() + .map_err(Clone::clone)? + .captures(message) + { + return Ok(Some(Self::Conversion { + source: captures[1].to_ascii_lowercase(), + target: captures[2].to_ascii_lowercase(), + })); + } + Ok(UNKNOWN_CAST_TYPE + .as_ref() + .map_err(Clone::clone)? + .captures(message) + .map(|captures| Self::UnknownTarget(captures[1].to_ascii_lowercase()))) + } + + fn rank(&self, site: &CastSite) -> u8 { + match self { + Self::Conversion { source, target } => { + let target_matches: bool = type_family(&site.target) == target; + let source_matches: bool = site.operand_words.contains(source); + match (target_matches, source_matches, site.literal_operand) { + (true, true, true) => 0, + (true, true, false) => 1, + (true, false, true) => 2, + _ => 3, + } + } + Self::UnknownTarget(name) if site.target.to_ascii_lowercase() == *name => 0, + Self::UnknownTarget(_) => 3, + } + } +} + +/// Literal sites for coercion findings, consumed in source order per finding kind. +struct LiteralSites { + comparisons: Vec<(usize, usize)>, + in_lists: Vec<(usize, usize)>, + temporals: Vec<(usize, usize)>, + intervals: Vec<(usize, usize)>, +} + +impl LiteralSites { + fn new(tokens: &[Token]) -> Self { + let mut sites: Self = Self { + comparisons: Vec::new(), + in_lists: Vec::new(), + temporals: Vec::new(), + intervals: Vec::new(), + }; + for (index, token) in tokens.iter().enumerate() { + let previous: Option<&Token> = index.checked_sub(1).map(|previous| &tokens[previous]); + let next: Option<&Token> = tokens.get(index + 1); + let through_next: (usize, usize) = ( + token.span.start, + next.map_or(token.span.end, |next| next.span.end), + ); + match token.token_type { + TokenType::String if is_comparison_operand(previous, next) => { + sites.comparisons.push((token.span.start, token.span.end)); + } + TokenType::In if next.is_some_and(|next| next.token_type == TokenType::LParen) => { + sites.in_lists.push((token.span.start, token.span.end)); + } + TokenType::Date | TokenType::Timestamp | TokenType::DateTime + if next.is_some_and(|next| next.token_type == TokenType::String) => + { + sites.temporals.push(through_next); + } + TokenType::Interval => sites.intervals.push(through_next), + _ => {} + } + } + sites.comparisons.reverse(); + sites.in_lists.reverse(); + sites.temporals.reverse(); + sites.intervals.reverse(); + sites + } + + fn next(&mut self, message: &str) -> Option<(usize, usize)> { + match message { + COMPARISON_LITERAL => self.comparisons.pop(), + IN_LIST_LITERAL => self.in_lists.pop(), + TEMPORAL_LITERAL => self.temporals.pop(), + INTERVAL_LITERAL => self.intervals.pop(), + _ => None, + } + } +} + +fn is_comparison_operand(previous: Option<&Token>, next: Option<&Token>) -> bool { + previous + .into_iter() + .chain(next) + .any(|neighbour| COMPARISON_OPERATORS.contains(&neighbour.token_type)) +} + +fn is_cast(code: &str) -> bool { + code == INVALID_CAST || code == LOSSY_CAST +} + +fn is_set_operation(code: &str) -> bool { + SET_OPERATION_CODES.contains(&code) +} + +fn cast_sites(tokens: &[Token]) -> Vec { + let mut sites: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + match token.token_type { + TokenType::Cast | TokenType::TryCast | TokenType::SafeCast + if tokens + .get(index + 1) + .is_some_and(|next| next.token_type == TokenType::LParen) => + { + let Some(close) = matching_paren(tokens, index + 1) else { + continue; + }; + let mut depth = 0usize; + let mut target_start = None; + for (offset, inner) in tokens[index + 2..close].iter().enumerate() { + match inner.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => depth = depth.saturating_sub(1), + TokenType::As if depth == 0 => target_start = Some(index + 3 + offset), + _ => {} + } + } + if let Some(target_start) = target_start { + let operand = &tokens[index + 2..target_start - 1]; + sites.push(CastSite { + start: token.span.start, + end: tokens[close].span.end, + target: type_text(&tokens[target_start..close]), + operand_words: operand_words(operand), + literal_operand: literal_operand(operand), + }); + } + } + TokenType::DColon => { + let Some(first) = tokens.get(index + 1) else { + continue; + }; + let mut last = index + 1; + if tokens + .get(index + 2) + .is_some_and(|next| next.token_type == TokenType::LParen) + && let Some(close) = matching_paren(tokens, index + 2) + { + last = close; + } + let operand = &tokens[index.saturating_sub(1)..index]; + sites.push(CastSite { + start: index + .checked_sub(1) + .map_or(token.span.start, |previous| tokens[previous].span.start), + end: tokens[last].span.end, + target: first.text.clone(), + operand_words: operand_words(operand), + literal_operand: literal_operand(operand), + }); + } + _ => {} + } + } + sites.sort_by_key(|site| site.start); + sites +} + +fn set_operation_sites(tokens: &[Token]) -> Vec { + let mut sites: Vec = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if !matches!( + token.token_type, + TokenType::Union | TokenType::Except | TokenType::Intersect + ) { + continue; + } + let Some(select) = tokens[index + 1..] + .iter() + .position(|next| next.token_type == TokenType::Select) + .map(|offset| index + 1 + offset) + else { + continue; + }; + if tokens[index + 1..select].iter().any(|between| { + !matches!( + between.token_type, + TokenType::All | TokenType::Distinct | TokenType::LParen + ) && !between.text.eq_ignore_ascii_case("by") + && !between.text.eq_ignore_ascii_case("name") + }) { + continue; + } + sites.push(SetOperationSite { + keyword: (token.span.start, token.span.end), + projections: projection_spans(tokens, select), + }); + } + sites +} + +fn matching_paren(tokens: &[Token], open: usize) -> Option { + let mut depth = 0usize; + for (index, token) in tokens.iter().enumerate().skip(open) { + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth -= 1; + if depth == 0 { + return Some(index); + } + } + _ => {} + } + } + None +} + +fn operand_words(tokens: &[Token]) -> Vec { + tokens + .iter() + .map(|token| type_family(&token.text).to_owned()) + .chain(tokens.iter().map(|token| token.text.to_ascii_lowercase())) + .collect() +} + +fn literal_operand(tokens: &[Token]) -> bool { + !tokens + .iter() + .any(|token| matches!(token.token_type, TokenType::Var | TokenType::Identifier)) +} + +fn type_text(tokens: &[Token]) -> String { + tokens + .first() + .map(|token| token.text.clone()) + .unwrap_or_default() +} + +fn type_family(type_name: &str) -> &'static str { + match type_name.to_ascii_uppercase().as_str() { + "INT" | "INTEGER" | "BIGINT" | "SMALLINT" | "TINYINT" | "HUGEINT" | "UBIGINT" + | "UINTEGER" | "USMALLINT" | "UTINYINT" | "INT2" | "INT4" | "INT8" | "BYTEINT" => "integer", + "NUMBER" | "NUMERIC" | "DECIMAL" | "FLOAT" | "DOUBLE" | "REAL" | "FLOAT4" | "FLOAT8" => { + "numeric" + } + "BOOLEAN" | "BOOL" => "boolean", + name if name.starts_with(TIMESTAMP_TYPE) || name == DATETIME_TYPE => "timestamp", + _ => "other", + } +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs index 4ae2af8033..bd969772d8 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests.rs @@ -6,3 +6,5 @@ mod positions; mod semantic_validation; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_type_locations.rs"] +mod type_locations; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_type_locations.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_type_locations.rs new file mode 100644 index 0000000000..06c47b5f23 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_type_locations.rs @@ -0,0 +1,116 @@ +use crate::semantic_validation::main::diagnostics::binding_diagnostics; +use crate::semantic_validation::tests::test_types::TypeLocationTestCase; + +#[test] +fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_expression() +-> Result<(), String> { + let test_cases = [ + TypeLocationTestCase { + description: "two literal casts land on their own casts", + sql: "SELECT CAST(TIMESTAMP '2026-04-01' AS INTEGER) AS a, \ + CAST(TIMESTAMP '2026-04-02' AS INTEGER) AS b FROM orders", + dialect: "duckdb", + findings: &[ + ("W213", "Cannot cast timestamp to integer"), + ("W213", "Cannot cast timestamp to integer"), + ], + expected_starts: &["CAST(TIMESTAMP '2026-04-01'", "CAST(TIMESTAMP '2026-04-02'"], + }, + TypeLocationTestCase { + description: "a literal cast skips an unrelated cast to the same type", + sql: "SELECT CAST(1 AS INTEGER) AS a UNION ALL \ + SELECT CAST(TIMESTAMP '2026-01-01' AS INTEGER)", + dialect: "duckdb", + findings: &[("E218", "Cannot cast timestamp to integer")], + expected_starts: &["CAST(TIMESTAMP"], + }, + TypeLocationTestCase { + description: "set-operation findings point at the branch projection", + sql: "SELECT 1 AS order_id, 'open' AS status UNION ALL SELECT 'x', 'closed' \ + UNION ALL SELECT 'y', 'void'", + dialect: "snowflake", + findings: &[ + ( + "W214", + "Set-operation column 1 may fail during runtime conversion: \ + accumulated type NUMBER, next type VARCHAR", + ), + ( + "W214", + "Set-operation column 1 may fail during runtime conversion: \ + accumulated type NUMBER, next type VARCHAR", + ), + ], + expected_starts: &["'x'", "'y'"], + }, + TypeLocationTestCase { + description: "coercible-literal comparisons land on their own literals", + sql: "SELECT 1 = 'not-a-number' AS first_result, 2 = 'not-a-number' AS second_result", + dialect: "duckdb", + findings: &[ + ("W213", "Comparison contains an invalid coercible literal"), + ("W213", "Comparison contains an invalid coercible literal"), + ], + expected_starts: &["'not-a-number' AS first", "'not-a-number' AS second"], + }, + TypeLocationTestCase { + description: "a set-operation arity finding points at the operator", + sql: "SELECT 1 AS a, 2 AS b EXCEPT SELECT 1", + dialect: "duckdb", + findings: &[( + "E216", + "Set-operation operands return different column counts: left 2, right 1", + )], + expected_starts: &["EXCEPT"], + }, + ]; + for test_case in &test_cases { + let rows = test_case + .findings + .iter() + .map(|(code, message)| { + ( + (*code).to_owned(), + (*message).to_owned(), + None, + None, + None, + None, + "error".to_owned(), + ) + }) + .collect(); + let mapped = binding_diagnostics(test_case.sql, test_case.dialect, rows)?; + let observed: Vec = mapped + .iter() + .zip(test_case.expected_starts) + .map(|(row, expected)| { + row.4.map_or_else(String::new, |start| { + test_case + .sql + .chars() + .skip(start) + .take(expected.chars().count()) + .collect() + }) + }) + .collect(); + assert_eq!( + mapped.len(), + test_case.expected_starts.len(), + "{}", + test_case.description + ); + assert_eq!( + observed, test_case.expected_starts, + "{}", + test_case.description + ); + assert!( + mapped.iter().all(|row| row.2.is_some() && row.3.is_some()), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs index cf71c10603..030a8221de 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/tests/test_types.rs @@ -37,3 +37,11 @@ pub(crate) struct UniqueWordScalingTestCase { pub(crate) identifiers: usize, pub(crate) expected_max_duration: std::time::Duration, } + +pub(crate) struct TypeLocationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) dialect: &'static str, + pub(crate) findings: &'static [(&'static str, &'static str)], + pub(crate) expected_starts: &'static [&'static str], +} diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs index 0824ad481c..7669306fe9 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{HashMap, HashSet, VecDeque}; use std::str::FromStr; @@ -256,7 +257,7 @@ const DEFAULT_RULES: [&str; 13] = [ INLINE_QUERY_RELATION.code, ]; -const ALL_RULE_METADATA: [&LintRuleMetadata; 41] = [ +const ALL_RULE_METADATA: [&LintRuleMetadata; 44] = [ &NULL_COMPARISON, &IMPLICIT_CARTESIAN_JOIN, &JOIN_WITHOUT_CONDITION, @@ -298,6 +299,9 @@ const ALL_RULE_METADATA: [&LintRuleMetadata; 41] = [ &INLINE_QUERY_RELATION, &JOIN_EXPRESSION, &FINAL_CTE_NAME, + &crate::sql_quality::constants::UNUSED_CTE_OUTPUT, + &crate::sql_quality::constants::RANKING_SORT_CAP, + &crate::sql_quality::constants::LONG_LITERAL, ]; fn is_ceremonial_cte_name(name: &str) -> bool { @@ -434,6 +438,22 @@ pub(crate) fn lint(request: LintRequest) -> Result { Some("renaming a CTE requires updating its references"), )); } + diagnostics.extend(crate::sql_quality::main::diagnostics::diagnostics( + &crate::sql_quality::models::QualityRequest { + statements: &statements, + tokens: &tokens, + enabled: &enabled, + max_literal_length: request + .max_literal_length + .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by: request + .max_ranking_order_by + .unwrap_or(crate::sql_quality::constants::DEFAULT_MAX_RANKING_ORDER_BY), + relation_keys: &request.relation_keys, + externally_referenced_ctes: &externally_referenced_ctes, + fixture_body: request.allows_ceremonial_select, + }, + )?); diagnostics.sort_by_key(|diagnostic| (diagnostic.start, diagnostic.end, diagnostic.code)); Ok(LintResponse { version: LINT_API_VERSION, @@ -1845,7 +1865,7 @@ fn diagnostic( LintDiagnostic { code: rule.code, message: rule.message, - remediation: rule.remediation, + remediation: Cow::Borrowed(rule.remediation), start: span.start, end: span.end, fix_unavailable_reason: fix.is_none().then_some(fix_unavailable_reason).flatten(), diff --git a/crates/sqlbuild-rules-native/src/sql_lint/mod.rs b/crates/sqlbuild-rules-native/src/sql_lint/mod.rs index 4d5986ec3e..e82420248a 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/mod.rs @@ -3,7 +3,7 @@ pub(crate) mod _helpers; mod constants; pub(crate) mod main; -mod models; +pub(crate) mod models; pub(crate) mod types; #[cfg(test)] diff --git a/crates/sqlbuild-rules-native/src/sql_lint/models.rs b/crates/sqlbuild-rules-native/src/sql_lint/models.rs index b1acb86dbd..9371ecd79c 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/models.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use polyglot_sql::tokens::Span; @@ -73,6 +74,12 @@ pub(crate) struct LintRequest { pub allows_dynamic_output_star: bool, #[serde(default)] pub allows_empty_fixture_star: bool, + #[serde(default)] + pub max_literal_length: Option, + #[serde(default)] + pub max_ranking_order_by: Option, + #[serde(default)] + pub relation_keys: HashMap>>, } #[derive(Debug)] @@ -94,7 +101,7 @@ pub(crate) struct LintResponse { pub(crate) struct LintDiagnostic { pub code: &'static str, pub message: &'static str, - pub remediation: &'static str, + pub remediation: Cow<'static, str>, pub start: usize, pub end: usize, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs index 971222a13d..2c8bcbbde8 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/tests/test_sql_lint.rs @@ -1420,10 +1420,10 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R expected_replacement: None, }, test_types::AdditionalLintRuleTestCase { - description: "ordered row number is clean", + description: "ordered row number without a declared key is unproven", sql: "SELECT ROW_NUMBER() OVER (PARTITION BY id ORDER BY created_at, id) FROM items", rule: "SQBRSQL018", - expected_anchor: None, + expected_anchor: Some("ROW_NUMBER"), expected_replacement: None, }, test_types::AdditionalLintRuleTestCase { diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs new file mode 100644 index 0000000000..37232ab85c --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/cte_columns.rs @@ -0,0 +1,457 @@ +//! SQBRSQL042: output columns of non-final CTEs that no later step reads. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use polyglot_sql::expressions::{Cte, JoinKind, Select}; +use polyglot_sql::scope::walk_in_scope; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::sql_lint::models::{LintDiagnostic, LintEdit}; +use crate::sql_quality::constants::{ + DISTINCT_UNUSED_REMEDIATION, GROUP_BY_ALL_UNUSED_REMEDIATION, GROUP_KEY_UNUSED_REMEDIATION, + INTERPOLATION_SENTINEL_PREFIXES, UNUSED_CTE_OUTPUT, +}; +use crate::sql_quality::models::QualityRequest; +use crate::sql_quality::removal_safety::{expression_refusal, reads_whole_rows, select_refusal}; +use crate::sql_quality::syntax::{ + Projection, columns, cte_select_tokens, lower, projection_removal, projections, qualifier, + sources, +}; +use crate::sql_tokens::main::projection_spans::projection_spans; + +const HARNESS_PREFIX: &str = "__"; +const COLUMNS_FUNCTION: &str = "COLUMNS"; +const LAYOUT_FIX_REFUSAL: &str = "The projection list could not be mapped to authored tokens"; + +/// What one later step reads from a CTE. +#[derive(Default)] +struct Usage { + names: HashSet, + everything: bool, + passes_to: Vec, + read: bool, +} + +impl Usage { + fn merged(mut self, other: Self) -> Self { + self.names.extend(other.names); + self.everything |= other.everything; + self.passes_to.extend(other.passes_to); + self.read |= other.read; + self + } +} + +/// One CTE whose unused outputs are being reported. +struct CteOutputs<'a> { + cte: &'a Cte, + name: &'a str, + usage: Option<&'a Usage>, +} + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + if request.fixture_body { + return Vec::new(); + } + let mut found: Vec = Vec::new(); + for statement in request.statements { + let Expression::Select(root) = statement else { + continue; + }; + let Some(with) = root.with.as_ref().filter(|with| !with.recursive) else { + continue; + }; + let mut terminal: Select = root.as_ref().clone(); + terminal.with = None; + let terminal: Expression = Expression::Select(Box::new(terminal)); + let names: Vec = with.ctes.iter().map(|cte| lower(&cte.alias.name)).collect(); + let used: HashMap = reads(&with.ctes, &names, &terminal); + let select_tokens: HashMap = cte_select_tokens(request.tokens); + for (index, cte) in with + .ctes + .iter() + .enumerate() + .take(with.ctes.len().saturating_sub(1)) + { + let name: &str = &names[index]; + if name.starts_with(HARNESS_PREFIX) || request.externally_referenced_ctes.contains(name) + { + continue; + } + found.extend(unused_outputs( + request, + &select_tokens, + CteOutputs { + cte, + name, + usage: used.get(name), + }, + )); + } + } + found +} + +/// Names each CTE's later readers use, following `SELECT *` pass-through to the reader. +fn reads(ctes: &[Cte], names: &[String], terminal: &Expression) -> HashMap { + let bodies: Vec<(Option, &Expression)> = ctes + .iter() + .map(|cte| (Some(lower(&cte.alias.name)), &cte.this)) + .chain(std::iter::once((None, terminal))) + .collect(); + let known: HashSet<&str> = names.iter().map(String::as_str).collect(); + let outputs: Vec>> = ctes.iter().map(output_names).collect(); + let mut readers: HashMap> = HashMap::new(); + for (index, (_, body)) in bodies.iter().enumerate() { + for name in read_relations(body) { + if known.contains(name.as_str()) { + readers.entry(name).or_default().push(index); + } + } + } + let mut usage: HashMap = HashMap::new(); + for (index, name) in names.iter().enumerate() { + let mut current: Usage = Usage::default(); + if index + 1 == names.len() { + current.everything = true; + } + for &reader in readers.get(name.as_str()).into_iter().flatten() { + if reader > index { + let (reader_name, body) = &bodies[reader]; + let target: Target<'_> = Target { + name, + outputs: outputs[index].as_ref(), + }; + current = current.merged(reader_usage(&target, reader_name.clone(), body)); + } + } + current.names.extend(self_reads(&ctes[index].this)); + usage.insert(name.clone(), current); + } + for index in (0..names.len()).rev() { + let passes: Vec = usage[&names[index]].passes_to.clone(); + for reader in passes { + let (everything, inherited): (bool, Vec) = + usage.get(&reader).map_or((true, Vec::new()), |next| { + (next.everything, next.names.iter().cloned().collect()) + }); + if let Some(current) = usage.get_mut(&names[index]) { + current.everything |= everything; + current.names.extend(inherited); + } + } + } + usage +} + +/// The output names of a CTE, or None when a star or unnamed projection hides them. +fn output_names(cte: &Cte) -> Option> { + let Expression::Select(select) = &cte.this else { + return None; + }; + projections(select) + .into_iter() + .map(|item| match item { + Projection::Named { name, .. } => Some(name), + _ => None, + }) + .collect() +} + +fn read_relations(body: &Expression) -> HashSet { + body.dfs() + .filter_map(|node| match node { + Expression::Table(table) => Some(lower(&table.name.name)), + _ => None, + }) + .collect() +} + +/// The CTE whose readers are being inspected. +struct Target<'a> { + name: &'a str, + outputs: Option<&'a HashSet>, +} + +fn reader_usage(target: &Target<'_>, reader: Option, body: &Expression) -> Usage { + let name: &str = target.name; + let mut usage: Usage = Usage::default(); + let mut aliases: HashSet = HashSet::new(); + let mut top_level_star: bool = false; + for node in body.dfs() { + let Expression::Select(select) = node else { + continue; + }; + let local: Vec = sources(select) + .into_iter() + .filter(|(_, relation, _)| relation.as_deref() == Some(name)) + .map(|(alias, _, _)| alias) + .collect(); + if local.is_empty() { + continue; + } + if select.joins.iter().any(|join| { + matches!( + join.kind, + JoinKind::Natural + | JoinKind::NaturalLeft + | JoinKind::NaturalRight + | JoinKind::NaturalFull + ) + }) { + usage.everything = true; + } + for join in &select.joins { + usage + .names + .extend(join.using.iter().map(|identifier| lower(&identifier.name))); + } + let top: bool = std::ptr::eq(node, body); + if reads_as_row(select, &local, target.outputs) { + usage.everything = true; + } + for projection in projections(select) { + if let Projection::Star(star) = projection + && star + .table + .as_ref() + .is_none_or(|table| local.contains(&lower(&table.name))) + { + if top + && reader.is_some() + && star.except.is_none() + && star.replace.is_none() + && !reads_whole_rows(select) + { + top_level_star = true; + } else { + usage.everything = true; + } + } + } + aliases.extend(local); + } + if aliases.is_empty() { + return usage; + } + usage.read = true; + usage.everything |= columns(body) + .into_iter() + .any(|column| is_interpolation(&column.name.name)); + if top_level_star && let Some(reader) = reader { + usage.passes_to.push(reader); + } + for column in body.dfs().filter_map(|node| match node { + Expression::Column(column) => Some(column), + _ => None, + }) { + if qualifier(column).is_none_or(|table| aliases.contains(&table)) { + usage.names.insert(lower(&column.name.name)); + } + } + usage +} + +/// Whether a reader consumes whole rows: a nested star, `COLUMNS(...)`, or the alias itself. +fn reads_as_row(select: &Select, local: &[String], outputs: Option<&HashSet>) -> bool { + for clause in clause_roots(select) { + for node in walk_in_scope(clause, false) { + if reads_row_at(select, node, local, outputs) { + return true; + } + } + } + false +} + +fn reads_row_at( + select: &Select, + node: &Expression, + local: &[String], + outputs: Option<&HashSet>, +) -> bool { + match node { + Expression::Star(star) => { + let projected: bool = select + .expressions + .iter() + .any(|item| std::ptr::eq(item, node)); + let table: Option = star.table.as_ref().map(|table| lower(&table.name)); + !projected && table.is_none_or(|table| local.contains(&table)) + } + Expression::BracedWildcard(_) => true, + Expression::Function(function) => function.name.eq_ignore_ascii_case(COLUMNS_FUNCTION), + Expression::Column(column) => { + let read: String = lower(&column.name.name); + let named_output: bool = outputs.is_some_and(|names| names.contains(&read)); + column.table.is_none() && local.contains(&read) && !named_output + } + _ => false, + } +} + +/// Root expressions of every clause of one select, without its FROM relations. +fn clause_roots(select: &Select) -> Vec<&Expression> { + let mut roots: Vec<&Expression> = select.expressions.iter().collect(); + roots.extend(select.where_clause.iter().map(|clause| &clause.this)); + roots.extend(select.having.iter().map(|clause| &clause.this)); + roots.extend(select.qualify.iter().map(|clause| &clause.this)); + if let Some(group) = &select.group_by { + roots.extend(group.expressions.iter()); + } + if let Some(order) = &select.order_by { + roots.extend(order.expressions.iter().map(|ordered| &ordered.this)); + } + roots.extend(select.joins.iter().filter_map(|join| join.on.as_ref())); + roots +} + +/// Lateral alias reuse: a later projection or a clause of the CTE reading an earlier output. +fn self_reads(body: &Expression) -> HashSet { + let mut reads: HashSet = HashSet::new(); + let Expression::Select(select) = body else { + return reads; + }; + let mut earlier: HashSet = HashSet::new(); + for item in projections(select) { + if let Projection::Named { name, expression } = item { + for read in unqualified_reads(expression) { + if earlier.contains(&read) && read != name { + reads.insert(read); + } + } + earlier.insert(name); + } + } + let mut clauses: Vec<&Expression> = Vec::new(); + clauses.extend(select.where_clause.iter().map(|clause| &clause.this)); + clauses.extend( + select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()), + ); + clauses.extend(select.having.iter().map(|clause| &clause.this)); + clauses.extend(select.qualify.iter().map(|clause| &clause.this)); + if let Some(order) = &select.order_by { + clauses.extend(order.expressions.iter().map(|ordered| &ordered.this)); + } + for clause in clauses { + reads.extend( + unqualified_reads(clause) + .into_iter() + .filter(|read| earlier.contains(read)), + ); + } + reads +} + +fn unqualified_reads(expression: &Expression) -> Vec { + columns(expression) + .into_iter() + .filter(|column| qualifier(column).is_none()) + .map(|column| lower(&column.name.name)) + .collect() +} + +fn unused_outputs( + request: &QualityRequest<'_>, + select_tokens: &HashMap, + outputs: CteOutputs<'_>, +) -> Vec { + let CteOutputs { cte, name, usage } = outputs; + let Expression::Select(select) = &cte.this else { + return Vec::new(); + }; + let Some(usage) = usage.filter(|usage| !usage.everything && usage.read) else { + return Vec::new(); + }; + if !cte.columns.is_empty() { + return Vec::new(); + } + let output: Vec> = projections(select); + if !output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + return Vec::new(); + } + let spans: Vec<(usize, usize)> = select_tokens + .get(name) + .map(|&select_token| projection_spans(request.tokens, select_token)) + .unwrap_or_default(); + let grouped: HashSet = select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()) + .flat_map(columns) + .map(|column| lower(&column.name.name)) + .collect(); + let select_refused: Option<&'static str> = select_refusal(select, output.len()); + let mut found: Vec = Vec::new(); + for (index, item) in output.iter().enumerate() { + let Projection::Named { + name: column, + expression, + } = item + else { + continue; + }; + if usage.names.contains(column) || is_interpolation(column) { + continue; + } + let grouping_key: bool = matches!(expression, Expression::Column(source) if grouped.contains(&lower(&source.name.name))); + let (start, end) = spans + .get(index) + .copied() + .filter(|_| spans.len() == output.len()) + .unwrap_or((0, 0)); + let refusal: Option<&'static str> = select_refused + .or_else(|| expression_refusal(expression)) + .or_else(|| (spans.len() != output.len()).then_some(LAYOUT_FIX_REFUSAL)); + let fix: Option = refusal + .is_none() + .then(|| projection_removal(&spans, index)) + .flatten() + .map(|(start, end)| LintEdit { + start, + end, + replacement: String::new(), + }); + found.push(LintDiagnostic { + code: UNUSED_CTE_OUTPUT.code, + message: UNUSED_CTE_OUTPUT.message, + remediation: Cow::Borrowed(if select.distinct { + DISTINCT_UNUSED_REMEDIATION + } else if select + .group_by + .as_ref() + .is_some_and(|group| group.all == Some(true)) + { + GROUP_BY_ALL_UNUSED_REMEDIATION + } else if grouping_key { + GROUP_KEY_UNUSED_REMEDIATION + } else { + UNUSED_CTE_OUTPUT.remediation + }), + start, + end, + fix_unavailable_reason: if fix.is_none() { + Some(refusal.unwrap_or(LAYOUT_FIX_REFUSAL)) + } else { + None + }, + fix, + }); + } + found +} + +/// Neutralized `@macro` / `@variable` interpolations can stand for any columns. +fn is_interpolation(name: &str) -> bool { + let name: String = lower(name); + INTERPOLATION_SENTINEL_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/keys.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/keys.rs new file mode 100644 index 0000000000..2e006e53b1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/keys.rs @@ -0,0 +1,437 @@ +//! Unique-key propagation through one model's relations, for ranking determinism proofs. + +use std::collections::HashMap; + +use polyglot_sql::Expression; +use polyglot_sql::expressions::{JoinKind, Literal, Select}; + +use crate::sql_quality::constants::FIRST_ROW; +use crate::sql_quality::models::QualityRequest; +use crate::sql_quality::syntax::{Projection, columns, lower, projections, qualifier, sources}; +use crate::sql_quality::types::{InputKey, Keys}; + +/// Keys known for relations by name: declared upstream keys and earlier CTE outputs. +pub(crate) struct KeyEnvironment { + pub(crate) declared: HashMap, + pub(crate) ctes: HashMap, + /// Per CTE, `(row number output, partition output columns)` for deduplication filters. + pub(crate) row_numbers: HashMap)>>, +} + +impl KeyEnvironment { + /// Output keys of a query expression, by output column name. + pub(crate) fn query_keys(&self, expression: &Expression) -> Keys { + match expression { + Expression::Select(select) => self.select_keys(select), + Expression::Subquery(subquery) => self.query_keys(&subquery.this), + Expression::Paren(paren) => self.query_keys(&paren.this), + Expression::Union(union) if !union.all => all_columns_key(&union.left), + Expression::Intersect(intersect) => self.query_keys(&intersect.left), + Expression::Except(except) => self.query_keys(&except.left), + _ => Vec::new(), + } + } + + fn relation_keys(&self, relation: &Expression) -> Keys { + match relation { + Expression::Table(table) => { + let name: String = lower(&table.name.name); + self.ctes + .get(&name) + .cloned() + .or_else(|| self.declared.get(&name).cloned()) + .unwrap_or_default() + } + Expression::Subquery(subquery) => self.query_keys(&subquery.this), + _ => Vec::new(), + } + } + + /// Keys of a select's joined and filtered input rows, before projection. + pub(crate) fn input_keys(&self, select: &Select) -> Vec { + if select + .from + .as_ref() + .is_none_or(|from| from.expressions.len() != 1) + { + return Vec::new(); + } + let found: Vec<(String, Option, &Expression)> = sources(select); + let Some((first_alias, _, first)) = found.first() else { + return Vec::new(); + }; + let mut keys: Vec = qualify(first_alias, self.relation_keys(first)); + keys.extend(qualify(first_alias, self.deduplicated_keys(select, first))); + for (join, (alias, _, relation)) in select.joins.iter().zip(found.iter().skip(1)) { + let right: Vec = qualify(alias, self.relation_keys(relation)); + let equated: Vec<(String, String)> = join_equalities(join, alias); + let left_columns: Vec<&str> = equated.iter().map(|(_, left)| left.as_str()).collect(); + let right_columns: Vec<&str> = + equated.iter().map(|(right, _)| right.as_str()).collect(); + let left_unique: bool = keys.iter().any(|key| key_within(key, &left_columns)); + let right_unique: bool = right.iter().any(|key| key_within(key, &right_columns)); + let mut joined: Vec = Vec::new(); + if right_unique + && !matches!( + join.kind, + JoinKind::Right | JoinKind::Full | JoinKind::Cross + ) + { + joined.extend(keys.iter().cloned()); + } + if left_unique && matches!(join.kind, JoinKind::Inner | JoinKind::Right) { + joined.extend(right.iter().cloned()); + } + keys = joined; + } + keys + } + + /// Input keys plus a `QUALIFY ROW_NUMBER() ... = n` partition, which holds only for output rows. + fn filtered_input_keys(&self, select: &Select) -> Vec { + let mut keys: Vec = self.input_keys(select); + if let Some(qualify_clause) = &select.qualify { + keys.extend(qualified_rank_keys(&qualify_clause.this, &sources(select))); + } + keys + } + + /// Partition keys of a `ROW_NUMBER()` output that this select filters to `= 1`. + fn deduplicated_keys(&self, select: &Select, relation: &Expression) -> Keys { + let Expression::Table(table) = relation else { + return Vec::new(); + }; + let Some(ranks) = self.row_numbers.get(&lower(&table.name.name)) else { + return Vec::new(); + }; + let filtered: Vec = select + .where_clause + .as_ref() + .map(|clause| first_row_filters(&clause.this)) + .unwrap_or_default(); + ranks + .iter() + .filter(|(rank, _)| filtered.contains(rank)) + .map(|(_, key)| key.clone()) + .collect() + } + + fn select_keys(&self, select: &Select) -> Keys { + let output: Vec> = projections(select); + let mut keys: Keys = Vec::new(); + if select.distinct + && output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + keys.push(output_names(&output)); + } + if let Some(group_by) = &select.group_by + && group_by.all.is_none() + && let Some(grain) = output_key_of(&group_by.expressions, &output) + { + keys.push(grain); + return keys; + } + let star: bool = output + .iter() + .any(|item| matches!(item, Projection::Star(_))); + for input in self.filtered_input_keys(select) { + if let Some(key) = project_key(&input, &output, star) { + keys.push(key); + } + } + keys + } +} + +fn qualify(alias: &str, keys: Keys) -> Vec { + keys.into_iter() + .map(|key| qualified_key(alias, key)) + .collect() +} + +fn qualified_key(alias: &str, key: Vec) -> InputKey { + key.into_iter() + .map(|column| (alias.to_owned(), column)) + .collect() +} + +fn key_within(key: &InputKey, columns: &[&str]) -> bool { + key.iter() + .all(|(_, column)| columns.contains(&column.as_str())) +} + +/// `(right column, left column)` pairs equated by a join's ON or USING condition. +fn join_equalities( + join: &polyglot_sql::expressions::Join, + right_alias: &str, +) -> Vec<(String, String)> { + let mut pairs: Vec<(String, String)> = join + .using + .iter() + .map(|identifier| (lower(&identifier.name), lower(&identifier.name))) + .collect(); + let mut pending: Vec<&Expression> = join.on.iter().collect(); + while let Some(expression) = pending.pop() { + match expression { + Expression::And(operation) => { + pending.push(&operation.left); + pending.push(&operation.right); + } + Expression::Paren(paren) => pending.push(&paren.this), + Expression::Eq(operation) => { + if let (Expression::Column(left), Expression::Column(right)) = + (&operation.left, &operation.right) + { + let (inner, outer) = if qualifier(left).as_deref() == Some(right_alias) { + (left, right) + } else { + (right, left) + }; + pairs.push((lower(&inner.name.name), lower(&outer.name.name))); + } + } + _ => {} + } + } + pairs +} + +fn qualified_rank_keys( + condition: &Expression, + found: &[(String, Option, &Expression)], +) -> Vec { + let Expression::Eq(operation) = condition else { + return Vec::new(); + }; + let window = match (&operation.left, &operation.right) { + (Expression::WindowFunction(window), Expression::Literal(_)) + | (Expression::Literal(_), Expression::WindowFunction(window)) => window, + _ => return Vec::new(), + }; + if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { + return Vec::new(); + } + let mut key: InputKey = InputKey::new(); + for expression in &window.over.partition_by { + let Expression::Column(column) = expression else { + return Vec::new(); + }; + let alias: String = qualifier(column) + .or_else(|| (found.len() == 1).then(|| found[0].0.clone())) + .unwrap_or_default(); + key.insert((alias, lower(&column.name.name))); + } + vec![key] +} + +fn output_names(output: &[Projection<'_>]) -> Vec { + output + .iter() + .filter_map(|item| match item { + Projection::Named { name, .. } => Some(name.clone()), + _ => None, + }) + .collect() +} + +fn output_key_of(expressions: &[Expression], output: &[Projection<'_>]) -> Option> { + let mut key: Vec = Vec::new(); + for expression in expressions { + if let Expression::Literal(literal) = expression + && let Literal::Number(text) = literal.as_ref() + { + let Ok(ordinal) = text.parse::() else { + return None; + }; + let Some(Projection::Named { name, .. }) = ordinal + .checked_sub(1) + .and_then(|position| output.get(position)) + else { + return None; + }; + key.push(name.clone()); + continue; + } + let Expression::Column(column) = expression else { + return None; + }; + let name: String = lower(&column.name.name); + let projected: Option = output.iter().find_map(|item| match item { + Projection::Named { + name: output_name, + expression: Expression::Column(source), + } if lower(&source.name.name) == name => Some(output_name.clone()), + Projection::Named { + name: output_name, .. + } if *output_name == name => Some(output_name.clone()), + _ => None, + }); + key.push(projected?); + } + Some(key) +} + +fn project_key(input: &InputKey, output: &[Projection<'_>], star: bool) -> Option> { + let mut key: Vec = Vec::new(); + for (alias, column) in input { + let projected: Option = output.iter().find_map(|item| match item { + Projection::Named { + name, + expression: Expression::Column(source), + } if lower(&source.name.name) == *column + && qualifier(source).is_none_or(|table| table == *alias) => + { + Some(name.clone()) + } + Projection::Star(star) + if star + .table + .as_ref() + .is_none_or(|table| lower(&table.name) == *alias) => + { + Some(column.clone()) + } + _ => None, + }); + match projected { + Some(name) => key.push(name), + None if star => key.push(column.clone()), + None => return None, + } + } + Some(key) +} + +fn all_columns_key(left: &Expression) -> Keys { + let Expression::Select(select) = left else { + return Vec::new(); + }; + let output: Vec> = projections(select); + if output + .iter() + .all(|item| matches!(item, Projection::Named { .. })) + { + vec![output_names(&output)] + } else { + Vec::new() + } +} + +/// Columns a ranking window partitions and orders by, generously resolved to input aliases. +pub(crate) fn window_columns( + partition_by: &[Expression], + order_by: &[&Expression], +) -> Vec<(Option, String)> { + partition_by + .iter() + .chain(order_by.iter().copied()) + .flat_map(columns) + .map(|column| (qualifier(column), lower(&column.name.name))) + .collect() +} + +/// Whether some input key is covered by the window's partition and order columns. +pub(crate) fn covers(keys: &[InputKey], window: &[(Option, String)]) -> bool { + keys.iter().any(|key| key_covered(key, window)) +} + +fn key_covered(key: &InputKey, window: &[(Option, String)]) -> bool { + key.iter() + .all(|(alias, column)| window_has(window, alias, column)) +} + +fn window_has(window: &[(Option, String)], alias: &str, column: &str) -> bool { + window.iter().any(|(qualifier, name)| { + name == column && qualifier.as_deref().is_none_or(|table| table == alias) + }) +} + +/// Keys of declared relations and of each top-level CTE, in definition order. +pub(crate) fn key_environment(request: &QualityRequest<'_>) -> KeyEnvironment { + let mut environment: KeyEnvironment = KeyEnvironment { + declared: lowered_keys(request.relation_keys), + ctes: HashMap::new(), + row_numbers: HashMap::new(), + }; + for statement in request.statements { + let Expression::Select(select) = statement else { + continue; + }; + for cte in select.with.iter().flat_map(|with| with.ctes.iter()) { + let keys = environment.query_keys(&cte.this); + environment.ctes.insert(lower(&cte.alias.name), keys); + if let Expression::Select(body) = &cte.this { + environment + .row_numbers + .insert(lower(&cte.alias.name), row_number_outputs(body)); + } + } + } + environment +} + +fn lowered_keys(keys: &HashMap) -> HashMap { + let mut lowered: HashMap = HashMap::new(); + for (name, relation_keys) in keys { + let mut converted: Keys = Vec::new(); + for key in relation_keys { + let mut columns: Vec = Vec::new(); + for column in key { + columns.push(lower(column)); + } + converted.push(columns); + } + lowered.insert(lower(name), converted); + } + lowered +} + +/// `(output name, partition output columns)` for each `ROW_NUMBER() OVER (PARTITION BY ...)` output. +fn row_number_outputs(select: &Select) -> Vec<(String, Vec)> { + let output: Vec> = projections(select); + let mut found: Vec<(String, Vec)> = Vec::new(); + for item in &output { + let Projection::Named { + name, + expression: Expression::WindowFunction(window), + } = item + else { + continue; + }; + if !matches!(window.this, Expression::RowNumber(_)) || window.over.partition_by.is_empty() { + continue; + } + if let Some(key) = output_key_of(&window.over.partition_by, &output) { + found.push((name.clone(), key)); + } + } + found +} + +/// Columns a filter pins to the first row, as in `WHERE rn = 1 AND ...`. +fn first_row_filters(condition: &Expression) -> Vec { + let mut found: Vec = Vec::new(); + let mut pending: Vec<&Expression> = vec![condition]; + while let Some(expression) = pending.pop() { + match expression { + Expression::And(operation) => { + pending.push(&operation.left); + pending.push(&operation.right); + } + Expression::Paren(paren) => pending.push(&paren.this), + Expression::Eq(operation) => { + if let (Expression::Column(column), Expression::Literal(literal)) + | (Expression::Literal(literal), Expression::Column(column)) = + (&operation.left, &operation.right) + && matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) + { + found.push(lower(&column.name.name)); + } + } + _ => {} + } + } + found +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/literals.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/literals.rs new file mode 100644 index 0000000000..667db34102 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/literals.rs @@ -0,0 +1,64 @@ +//! SQBRSQL044: string literals longer than the project limit. + +use std::borrow::Cow; + +use polyglot_sql::tokens::TokenType; + +use crate::configuration::main::setting_help::setting_help; +use crate::sql_lint::models::LintDiagnostic; +use crate::sql_quality::constants::{ + LONG_LITERAL, LONG_LITERAL_GUIDANCE, MAX_LITERAL_LENGTH_KEY, RULES_THRESHOLDS_SECTION, +}; +use crate::sql_quality::models::QualityRequest; + +const CONSTANT_FIX_REFUSAL: &str = + "Moving the value to a constant requires choosing its name and placement"; + +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { + request + .tokens + .iter() + .filter(|token| is_string(token.token_type)) + .filter_map(|token| { + let length: usize = token.text.chars().count(); + (length > request.max_literal_length).then(|| LintDiagnostic { + code: LONG_LITERAL.code, + message: LONG_LITERAL.message, + remediation: Cow::Owned(remediation(length, request.max_literal_length)), + start: token.span.start, + end: token.span.end, + fix: None, + fix_unavailable_reason: Some(CONSTANT_FIX_REFUSAL), + }) + }) + .collect() +} + +fn remediation(length: usize, limit: usize) -> String { + format!( + "{LONG_LITERAL_GUIDANCE} {}", + setting_help( + &format!("To allow literals of {length} characters (the current value is {limit})"), + RULES_THRESHOLDS_SECTION, + MAX_LITERAL_LENGTH_KEY, + &length.to_string(), + ) + ) +} + +fn is_string(kind: TokenType) -> bool { + matches!( + kind, + TokenType::String + | TokenType::DollarString + | TokenType::EscapeString + | TokenType::RawString + | TokenType::NationalString + | TokenType::UnicodeString + | TokenType::ByteString + | TokenType::TripleSingleQuotedString + | TokenType::TripleDoubleQuotedString + | TokenType::HeredocString + | TokenType::HeredocStringAlternative + ) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/ranking.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/ranking.rs new file mode 100644 index 0000000000..eceee81cfa --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/ranking.rs @@ -0,0 +1,216 @@ +//! SQBRSQL043 ranking sort caps and SQBRSQL018 ranking determinism proofs. + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; + +use polyglot_sql::expressions::{Over, Select, WindowFunction}; +use polyglot_sql::scope::walk_in_scope; +use polyglot_sql::tokens::Token; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::configuration::main::setting_help::setting_help; +use crate::sql_lint::models::LintDiagnostic; +use crate::sql_quality::constants::{ + MAX_RANKING_ORDER_BY_KEY, RANKING_SORT_CAP, RANKING_SORT_GUIDANCE, RULES_THRESHOLDS_SECTION, + TIE_SENSITIVE_FUNCTIONS, UNPROVEN_RANKING_MESSAGE, UNPROVEN_RANKING_REMEDIATION, + UNSTABLE_ROW_NUMBER_CODE, +}; +use crate::sql_quality::keys::{KeyEnvironment, covers, window_columns}; +use crate::sql_quality::models::QualityRequest; +use crate::sql_quality::syntax::{columns, lower}; +use crate::sql_quality::types::InputKey; + +const PRIORITY_FIX_REFUSAL: &str = + "Choosing a priority column and tie-breaker requires author intent"; +const KEY_FIX_REFUSAL: &str = "Declaring a key or adding a tie-breaker requires author intent"; + +struct RankingWindow<'a> { + function: &'static str, + window: &'a WindowFunction, + select: &'a Select, +} + +pub(crate) fn diagnostics( + request: &QualityRequest<'_>, + environment: &KeyEnvironment, +) -> Vec { + let cap: bool = request.enabled.contains(RANKING_SORT_CAP.code); + let proof: bool = request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE); + let mut used: HashSet = HashSet::new(); + let mut function_tokens: HashMap> = HashMap::new(); + for (index, token) in request.tokens.iter().enumerate() { + if TIE_SENSITIVE_FUNCTIONS + .iter() + .chain(["RANK", "DENSE_RANK"].iter()) + .any(|name| token.text.eq_ignore_ascii_case(name)) + { + function_tokens + .entry(token.text.to_ascii_uppercase()) + .or_default() + .push(index); + } + } + let mut found: Vec = Vec::new(); + for ranking in ranking_windows(request.statements) { + let over: Option<&Over> = effective_over(&ranking.window.over, ranking.select); + let order: Vec<&Expression> = over.map_or_else(Vec::new, |over| { + over.order_by.iter().map(|ordered| &ordered.this).collect() + }); + let partition: &[Expression] = over.map_or(&[], |over| over.partition_by.as_slice()); + let exceeds: bool = cap && order.len() > request.max_ranking_order_by; + let unproven: bool = proof + && !order.is_empty() + && TIE_SENSITIVE_FUNCTIONS.contains(&ranking.function) + && !proven(environment, ranking.select, partition, &order); + if !exceeds && !unproven { + continue; + } + let candidates: &[usize] = function_tokens + .get(ranking.function) + .map_or(&[], Vec::as_slice); + let token: Option = function_token( + request.tokens, + candidates, + window_anchor(partition, &order), + &used, + ); + let (start, end) = token.map_or((0, 0), |index| { + ( + request.tokens[index].span.start, + request.tokens[index].span.end, + ) + }); + used.extend(token); + if exceeds { + found.push(LintDiagnostic { + code: RANKING_SORT_CAP.code, + message: RANKING_SORT_CAP.message, + remediation: Cow::Owned(sort_cap_remediation( + order.len(), + request.max_ranking_order_by, + )), + start, + end, + fix: None, + fix_unavailable_reason: Some(PRIORITY_FIX_REFUSAL), + }); + } + if unproven { + found.push(LintDiagnostic { + code: UNSTABLE_ROW_NUMBER_CODE, + message: UNPROVEN_RANKING_MESSAGE, + remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), + start, + end, + fix: None, + fix_unavailable_reason: Some(KEY_FIX_REFUSAL), + }); + } + } + found +} + +fn sort_cap_remediation(sort_keys: usize, limit: usize) -> String { + format!( + "{RANKING_SORT_GUIDANCE} {}", + setting_help( + &format!("To allow {sort_keys} sort keys (the current value is {limit})"), + RULES_THRESHOLDS_SECTION, + MAX_RANKING_ORDER_BY_KEY, + &sort_keys.to_string(), + ) + ) +} + +fn proven( + environment: &KeyEnvironment, + select: &Select, + partition: &[Expression], + order: &[&Expression], +) -> bool { + let keys: Vec = environment.input_keys(select); + covers(&keys, &window_columns(partition, order)) +} + +fn ranking_windows(statements: &[Expression]) -> Vec> { + let mut found: Vec> = Vec::new(); + for statement in statements { + for node in statement.dfs() { + let Expression::Select(select) = node else { + continue; + }; + for local in walk_in_scope(node, false) { + let Expression::WindowFunction(window) = local else { + continue; + }; + let function: &'static str = match &window.this { + Expression::RowNumber(_) => "ROW_NUMBER", + Expression::Rank(_) => "RANK", + Expression::DenseRank(_) => "DENSE_RANK", + Expression::FirstValue(_) => "FIRST_VALUE", + Expression::LastValue(_) => "LAST_VALUE", + _ => continue, + }; + found.push(RankingWindow { + function, + window, + select, + }); + } + } + } + found +} + +fn effective_over<'a>(over: &'a Over, select: &'a Select) -> Option<&'a Over> { + let mut current: &Over = over; + let mut seen: HashSet = HashSet::new(); + loop { + if !current.order_by.is_empty() || current.window_name.is_none() { + return Some(current); + } + let name: String = lower(¤t.window_name.as_ref()?.name); + if !seen.insert(name.clone()) { + return None; + } + current = &select + .windows + .iter() + .flatten() + .find(|window| lower(&window.name.name) == name)? + .spec; + } +} + +/// Start offset of the first located column a window partitions or orders by. +fn window_anchor(partition: &[Expression], order: &[&Expression]) -> Option { + let mut expressions: Vec<&Expression> = partition.iter().collect(); + expressions.extend(order.iter().copied()); + for expression in expressions { + for column in columns(expression) { + if let Some(span) = column.span { + return Some(span.start); + } + } + } + None +} + +/// The function-name token before the anchor, or the first token not reported yet. +fn function_token( + tokens: &[Token], + candidates: &[usize], + anchor: Option, + used: &HashSet, +) -> Option { + if let Some(anchor) = anchor { + let before: usize = candidates.partition_point(|&index| tokens[index].span.start < anchor); + if let Some(position) = before.checked_sub(1) { + return Some(candidates[position]); + } + } + candidates + .iter() + .copied() + .find(|index| !used.contains(index)) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/removal_safety.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/removal_safety.rs new file mode 100644 index 0000000000..0de0113fd9 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/removal_safety.rs @@ -0,0 +1,99 @@ +//! SQBRSQL042 allow-list: when removing an unused CTE output provably keeps every row. + +use polyglot_sql::expressions::Select; +use polyglot_sql::traversal::contains_aggregate; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::sql_quality::constants::{ + DETERMINISTIC_SCALAR_FUNCTIONS, MIN_REMOVABLE_OUTPUTS, ROW_NEUTRAL_EXPRESSIONS, +}; + +const ORDER_BY_ALL: &str = "all"; +const DISTINCT_REFUSAL: &str = "SELECT DISTINCT depends on every output column"; +const DISTINCT_ON_REFUSAL: &str = "SELECT DISTINCT ON keeps one row per key chosen by the select"; +const GROUP_BY_ALL_REFUSAL: &str = "GROUP BY ALL groups by every non-aggregate output column"; +const ORDER_BY_ALL_REFUSAL: &str = "ORDER BY ALL sorts by every output column"; +const UNGROUPED_AGGREGATE_REFUSAL: &str = + "An aggregate select without GROUP BY depends on its whole projection"; +const POSITIONAL_REFUSAL: &str = "Positional GROUP BY or ORDER BY would shift"; +const ONLY_OUTPUT_REFUSAL: &str = "Removing the only output column leaves an empty select"; +const EXPRESSION_REFUSAL: &str = "Only column references and deterministic scalar expressions are removed automatically; this expression may change the rows the CTE returns"; + +/// Why removing any output of `select` could change its rows, if it could. +pub(super) fn select_refusal(select: &Select, outputs: usize) -> Option<&'static str> { + if select.distinct { + Some(DISTINCT_REFUSAL) + } else if select.distinct_on.is_some() { + Some(DISTINCT_ON_REFUSAL) + } else if select + .group_by + .as_ref() + .is_some_and(|group| group.all == Some(true)) + { + Some(GROUP_BY_ALL_REFUSAL) + } else if orders_by_all(select) { + Some(ORDER_BY_ALL_REFUSAL) + } else if select.group_by.is_none() + && (select.having.is_some() || select.expressions.iter().any(contains_aggregate)) + { + Some(UNGROUPED_AGGREGATE_REFUSAL) + } else if positional_clauses(select) { + Some(POSITIONAL_REFUSAL) + } else if outputs < MIN_REMOVABLE_OUTPUTS { + Some(ONLY_OUTPUT_REFUSAL) + } else { + None + } +} + +/// Why removing this one projection could change the rows, if it could. +pub(super) fn expression_refusal(expression: &Expression) -> Option<&'static str> { + (!row_neutral(expression)).then_some(EXPRESSION_REFUSAL) +} + +/// Whether a star select depends on every input column or on their positions. +pub(super) fn reads_whole_rows(select: &Select) -> bool { + select.distinct + || select.distinct_on.is_some() + || select.group_by.is_some() + || orders_by_all(select) + || positional_clauses(select) +} + +/// A column reference or deterministic scalar expression (no aggregate, window, SRF or subquery). +fn row_neutral(expression: &Expression) -> bool { + expression.dfs().all(|node| match node { + Expression::Function(function) => { + let name: String = function.name.to_ascii_uppercase(); + !function.distinct && DETERMINISTIC_SCALAR_FUNCTIONS.contains(&name.as_str()) + } + other => ROW_NEUTRAL_EXPRESSIONS.contains(&other.variant_name()), + }) +} + +fn orders_by_all(select: &Select) -> bool { + select + .order_by + .iter() + .flat_map(|order| order.expressions.iter()) + .any(|ordered| { + matches!(&ordered.this, Expression::Column(column) + if column.table.is_none() + && !column.name.quoted + && column.name.name.eq_ignore_ascii_case(ORDER_BY_ALL)) + }) +} + +fn positional_clauses(select: &Select) -> bool { + let numeric = |expression: &Expression| matches!(expression, Expression::Literal(_)); + select + .group_by + .iter() + .flat_map(|group| group.expressions.iter()) + .any(numeric) + || select + .order_by + .iter() + .flat_map(|order| order.expressions.iter()) + .any(|ordered| numeric(&ordered.this)) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/_helpers/syntax.rs b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/syntax.rs new file mode 100644 index 0000000000..bb52d54da7 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/_helpers/syntax.rs @@ -0,0 +1,125 @@ +//! Shared syntax helpers for SQL quality Rules. + +use std::collections::HashMap; + +use polyglot_sql::expressions::{Column, Select, Star}; +use polyglot_sql::tokens::{Token, TokenType}; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::sql_quality::constants::MIN_REMOVABLE_OUTPUTS; + +/// One projection of a select list. +pub(super) enum Projection<'a> { + Named { + name: String, + expression: &'a Expression, + }, + Star(&'a Star), + Unnamed, +} + +pub(super) fn lower(name: &str) -> String { + name.to_ascii_lowercase() +} + +pub(super) fn projections(select: &Select) -> Vec> { + select + .expressions + .iter() + .map(|expression| match expression { + Expression::Alias(alias) => Projection::Named { + name: lower(&alias.alias.name), + expression: &alias.this, + }, + Expression::Column(column) => Projection::Named { + name: lower(&column.name.name), + expression, + }, + Expression::Star(star) => Projection::Star(star), + _ => Projection::Unnamed, + }) + .collect() +} + +pub(super) fn columns(expression: &Expression) -> Vec<&Column> { + expression + .dfs() + .filter_map(|node| match node { + Expression::Column(column) => Some(column.as_ref()), + _ => None, + }) + .collect() +} + +pub(super) fn qualifier(column: &Column) -> Option { + column.table.as_ref().map(|table| lower(&table.name)) +} + +/// Relations read directly by one select: `(alias, relation name, expression)`. +pub(super) fn sources(select: &Select) -> Vec<(String, Option, &Expression)> { + let mut found: Vec<(String, Option, &Expression)> = Vec::new(); + let from = select.from.iter().flat_map(|from| from.expressions.iter()); + for expression in from.chain(select.joins.iter().map(|join| &join.this)) { + match expression { + Expression::Table(table) => { + let name: String = lower(&table.name.name); + let alias: String = table + .alias + .as_ref() + .map_or_else(|| name.clone(), |alias| lower(&alias.name)); + found.push((alias, Some(name), expression)); + } + Expression::Subquery(subquery) => { + let alias: String = subquery + .alias + .as_ref() + .map_or_else(String::new, |alias| lower(&alias.name)); + found.push((alias, None, expression)); + } + _ => found.push((String::new(), None, expression)), + } + } + found +} + +/// Token index of the SELECT opening each `name AS (SELECT` CTE body, by lower-cased name. +pub(super) fn cte_select_tokens(tokens: &[Token]) -> HashMap { + let significant: Vec = (0..tokens.len()) + .filter(|&index| !is_trivia(&tokens[index])) + .collect(); + let mut found: HashMap = HashMap::new(); + for window in significant.windows(4) { + let [name_index, as_index, paren_index, select_index] = window else { + continue; + }; + if tokens[*as_index].token_type == TokenType::As + && tokens[*paren_index].token_type == TokenType::LParen + && tokens[*select_index].token_type == TokenType::Select + { + found + .entry(lower(&tokens[*name_index].text)) + .or_insert(*select_index); + } + } + found +} + +/// The comma-inclusive deletion span that removes projection `index` from `spans`. +pub(super) fn projection_removal(spans: &[(usize, usize)], index: usize) -> Option<(usize, usize)> { + if spans.len() < MIN_REMOVABLE_OUTPUTS { + return None; + } + let (start, end) = spans[index]; + if index + 1 < spans.len() { + Some((start, spans[index + 1].0)) + } else { + Some((spans[index - 1].1, end)) + } +} + +pub(super) fn is_trivia(token: &Token) -> bool { + matches!( + token.token_type, + TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment + ) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/constants.rs b/crates/sqlbuild-rules-native/src/sql_quality/constants.rs new file mode 100644 index 0000000000..618d9afbe7 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/constants.rs @@ -0,0 +1,188 @@ +use crate::sql_lint::models::LintRuleMetadata; + +pub(crate) const UNUSED_CTE_OUTPUT: LintRuleMetadata = LintRuleMetadata { + code: "SQBRSQL042", + message: "CTE output column is never read later in the model", + remediation: "Remove the column from the CTE output, or read it in a later CTE or the final query.", +}; +pub(crate) const RANKING_SORT_CAP: LintRuleMetadata = LintRuleMetadata { + code: "SQBRSQL043", + message: "Ranking window sorts by more expressions than max_ranking_order_by", + remediation: "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker.", +}; +pub(crate) const LONG_LITERAL: LintRuleMetadata = LintRuleMetadata { + code: "SQBRSQL044", + message: "String literal is longer than max_literal_length", + remediation: "Define the value once as a constant and reference it with `@const`, or shorten it.", +}; +pub(crate) const UNSTABLE_ROW_NUMBER_CODE: &str = "SQBRSQL018"; +pub(crate) const UNPROVEN_RANKING_MESSAGE: &str = + "Ranking window order is not proven unique within its partition"; +pub(crate) const UNPROVEN_RANKING_REMEDIATION: &str = "PARTITION BY and ORDER BY must together cover a declared key of the ranked rows. End the ORDER BY with a unique tie-breaker column, or declare the key upstream. A key qualifies when it is a `unique` column audit on a column that is also non-null (a `not_null` audit or `nullable false`), or the `unique_key` of an incremental `merge` model or of a `delete_insert` model without a cursor; snapshot, append, microbatch and cursor-ranged `unique_key`s do not prove one row per key. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`), declare it with a `unique` audit, and order by it last."; +/// SQBRSQL043 guidance before the `max_ranking_order_by` setting snippet. +pub(crate) const RANKING_SORT_GUIDANCE: &str = "Order by the one or two columns that decide which row wins, then a unique row key as the final tie-breaker. If the rows have no unique column, build one in the upstream model from the columns that identify a row (for example `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` audit, so SQBRSQL018 can prove the order is deterministic. If the order genuinely needs more keys, raise `max_ranking_order_by` for the project."; +/// SQBRSQL044 guidance before the `max_literal_length` setting snippet. +pub(crate) const LONG_LITERAL_GUIDANCE: &str = "Define the value once as a constant and reference it with `@const`, in the narrowest scope that covers every use. (1) Only this model: add `constants (_status_pattern '')` to its `MODEL (...)` header and write `@const(\"_status_pattern\")`. (2) Other files use the same value: declare `CONSTANT (name status_pattern, value '');` in `_sqlbuild/_constants/` of the nearest folder that contains every use when they all sit directly in that folder, or in its `_sqlbuild/constants/` to include subfolders, and write `@const(\"status_pattern\")`. (3) Use top-level `constants/` only when the value is needed project-wide. Otherwise shorten the literal."; +pub(crate) const RULES_THRESHOLDS_SECTION: &str = "rules.thresholds"; +pub(crate) const MAX_RANKING_ORDER_BY_KEY: &str = "max_ranking_order_by"; +pub(crate) const MAX_LITERAL_LENGTH_KEY: &str = "max_literal_length"; +pub(crate) const DISTINCT_UNUSED_REMEDIATION: &str = "This CTE uses SELECT DISTINCT, so the column still decides which rows are kept. Make the deduplication independent of the column, or restructure the CTE so it outputs only what later steps read."; +pub(crate) const GROUP_KEY_UNUSED_REMEDIATION: &str = "Drop the column from the CTE output but keep it in GROUP BY; the grouping grain stays the same."; +pub(crate) const GROUP_BY_ALL_UNUSED_REMEDIATION: &str = "This CTE uses GROUP BY ALL, so the column is part of the grouping grain. List the grouping columns explicitly and keep this one in GROUP BY, then drop it from the output."; +/// Expression kinds that compute one value per input row without changing the row set. +pub(crate) const ROW_NEUTRAL_EXPRESSIONS: &[&str] = &[ + "literal", + "boolean", + "null", + "identifier", + "column", + "paren", + "dot", + "subscript", + "data_type", + "interval", + "cast", + "try_cast", + "safe_cast", + "collation", + "case", + "and", + "or", + "not", + "neg", + "add", + "sub", + "mul", + "div", + "mod", + "eq", + "neq", + "lt", + "lte", + "gt", + "gte", + "like", + "i_like", + "is", + "is_null", + "is_true", + "is_false", + "in", + "between", + "null_safe_eq", + "bitwise_and", + "bitwise_or", + "bitwise_xor", + "concat", + "concat_ws", + "substring", + "upper", + "lower", + "length", + "trim", + "replace", + "left", + "right", + "lpad", + "rpad", + "split", + "split_part", + "regexp_replace", + "initcap", + "position", + "contains", + "starts_with", + "ends_with", + "abs", + "round", + "floor", + "ceil", + "greatest", + "least", + "coalesce", + "null_if", + "if_func", + "if_null", + "nvl", + "nvl2", + "decode", + "date_add", + "date_diff", + "date_trunc", + "timestamp_trunc", + "extract", + "to_date", + "to_timestamp", + "to_char", + "to_number", + "date", + "year", + "month", + "last_day", + "struct_extract", +]; +/// Generic function names known to be deterministic scalar functions in supported dialects. +pub(crate) const DETERMINISTIC_SCALAR_FUNCTIONS: &[&str] = &[ + "ABS", + "ARRAY_SIZE", + "CEIL", + "CHARINDEX", + "COALESCE", + "CONCAT", + "CONCAT_WS", + "DATEADD", + "DATEDIFF", + "DATE_ADD", + "DATE_DIFF", + "DATE_PART", + "DATE_TRUNC", + "DAY", + "DIV0", + "FLOOR", + "GREATEST", + "HOUR", + "IFF", + "IFNULL", + "LEAST", + "LEFT", + "LENGTH", + "LOWER", + "LPAD", + "LTRIM", + "MD5", + "MOD", + "MONTH", + "NULLIF", + "NVL", + "NVL2", + "POWER", + "REGEXP_REPLACE", + "REPLACE", + "RIGHT", + "ROUND", + "RPAD", + "RTRIM", + "SPLIT_PART", + "SQRT", + "STRPOS", + "SUBSTR", + "SUBSTRING", + "TO_CHAR", + "TO_DATE", + "TO_VARCHAR", + "TRIM", + "TRUNC", + "UPPER", + "YEAR", + "ZEROIFNULL", +]; +pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: usize = 100; +pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: usize = 5; +pub(crate) const TIE_SENSITIVE_FUNCTIONS: [&str; 3] = ["ROW_NUMBER", "FIRST_VALUE", "LAST_VALUE"]; +pub(crate) const MIN_REMOVABLE_OUTPUTS: usize = 2; +pub(crate) const INTERPOLATION_SENTINEL_PREFIXES: [&str; 3] = [ + "__sqb_lint_", + "__sqlbuild_audit_parameter_", + "__sqlbuild_context_parameter_", +]; +pub(crate) const FIRST_ROW: &str = "1"; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/main/diagnostics.rs b/crates/sqlbuild-rules-native/src/sql_quality/main/diagnostics.rs new file mode 100644 index 0000000000..510d285a08 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/main/diagnostics.rs @@ -0,0 +1,28 @@ +use crate::sql_lint::models::LintDiagnostic; +use crate::sql_quality::constants::{ + LONG_LITERAL, RANKING_SORT_CAP, UNSTABLE_ROW_NUMBER_CODE, UNUSED_CTE_OUTPUT, +}; +use crate::sql_quality::keys::{KeyEnvironment, key_environment}; +use crate::sql_quality::models::QualityRequest; +use crate::sql_quality::{cte_columns, literals, ranking}; + +/// Run every enabled SQL quality Rule over one parsed body. +pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Result, String> { + if request.max_literal_length == 0 || request.max_ranking_order_by == 0 { + return Err("max_literal_length and max_ranking_order_by must be positive".to_owned()); + } + let mut found: Vec = Vec::new(); + if request.enabled.contains(LONG_LITERAL.code) { + found.extend(literals::diagnostics(request)); + } + if request.enabled.contains(UNUSED_CTE_OUTPUT.code) { + found.extend(cte_columns::diagnostics(request)); + } + if request.enabled.contains(RANKING_SORT_CAP.code) + || request.enabled.contains(UNSTABLE_ROW_NUMBER_CODE) + { + let environment: KeyEnvironment = key_environment(request); + found.extend(ranking::diagnostics(request, &environment)); + } + Ok(found) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs new file mode 100644 index 0000000000..f58548d888 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod diagnostics; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/mod.rs b/crates/sqlbuild-rules-native/src/sql_quality/mod.rs new file mode 100644 index 0000000000..bfc10329d3 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/mod.rs @@ -0,0 +1,21 @@ +//! Enforceable SQL quality Rules with project-wide limits and ranking determinism proofs. + +pub(crate) mod constants; +#[path = "_helpers/cte_columns.rs"] +mod cte_columns; +#[path = "_helpers/keys.rs"] +mod keys; +#[path = "_helpers/literals.rs"] +mod literals; +pub(crate) mod main; +pub(crate) mod models; +#[path = "_helpers/ranking.rs"] +mod ranking; +#[path = "_helpers/removal_safety.rs"] +mod removal_safety; +#[path = "_helpers/syntax.rs"] +mod syntax; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/models.rs b/crates/sqlbuild-rules-native/src/sql_quality/models.rs new file mode 100644 index 0000000000..4cc049d230 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/models.rs @@ -0,0 +1,17 @@ +use std::collections::{HashMap, HashSet}; + +use polyglot_sql::Expression; +use polyglot_sql::tokens::Token; + +/// Inputs for one body's quality Rules, borrowed from the native lint request. +#[derive(Debug)] +pub(crate) struct QualityRequest<'a> { + pub(crate) statements: &'a [Expression], + pub(crate) tokens: &'a [Token], + pub(crate) enabled: &'a HashSet, + pub(crate) max_literal_length: usize, + pub(crate) max_ranking_order_by: usize, + pub(crate) relation_keys: &'a HashMap>>, + pub(crate) externally_referenced_ctes: &'a HashSet, + pub(crate) fixture_body: bool, +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests.rs b/crates/sqlbuild-rules-native/src/sql_quality/tests.rs new file mode 100644 index 0000000000..9eac086363 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_quality_rules.rs"] +mod quality_rules; +#[path = "tests/test_quality_scale.rs"] +mod quality_scale; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/helpers.rs b/crates/sqlbuild-rules-native/src/sql_quality/tests/helpers.rs new file mode 100644 index 0000000000..68769c0765 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/tests/helpers.rs @@ -0,0 +1,92 @@ +use serde_json::{Value, json}; + +use crate::sql_lint::main::engine::lint_json; +use crate::sql_quality::tests::test_types::QualityRuleTestCase; + +pub(crate) fn lint(test_case: &QualityRuleTestCase) -> Result, String> { + let relation_keys: Value = + serde_json::from_str(test_case.relation_keys).map_err(|error| error.to_string())?; + let request: Value = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": [test_case.rule], + "max_literal_length": 24, + "max_ranking_order_by": 2, + "relation_keys": relation_keys, + }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default()) +} + +pub(crate) fn anchors(sql: &str, diagnostics: &[Value]) -> Vec { + diagnostics + .iter() + .map(|diagnostic| { + let start: usize = diagnostic["start"].as_u64().unwrap_or(0) as usize; + let end: usize = diagnostic["end"].as_u64().unwrap_or(0) as usize; + sql.chars() + .skip(start) + .take(end.saturating_sub(start)) + .collect() + }) + .collect() +} + +pub(crate) fn fixed(sql: &str, diagnostics: &[Value]) -> Option { + let mut edits: Vec<(usize, usize, String)> = diagnostics + .iter() + .filter_map(|diagnostic| { + let fix: &Value = diagnostic.get("fix")?; + Some(( + fix["start"].as_u64()? as usize, + fix["end"].as_u64()? as usize, + fix["replacement"].as_str()?.to_owned(), + )) + }) + .collect(); + edits.sort_by_key(|edit| std::cmp::Reverse(edit.0)); + let applied: bool = !edits.is_empty(); + let mut characters: Vec = sql.chars().collect(); + for (start, end, replacement) in edits { + characters.splice(start..end, replacement.chars()); + } + applied.then(|| characters.into_iter().collect()) +} + +/// A chain of CTEs that each drop their last column and rank without a declared key. +pub(crate) fn wide_cte_chain(cte_count: usize, column_count: usize) -> String { + let columns: Vec = (0..column_count).map(|index| format!("c{index}")).collect(); + let mut ctes: Vec = vec![format!( + "step_0 AS (SELECT {} FROM source_rows AS s)", + columns + .iter() + .map(|column| format!("s.{column}")) + .collect::>() + .join(", ") + )]; + for step in 1..cte_count { + let mut outputs: Vec = columns[..column_count - 1] + .iter() + .map(|column| format!("p.{column}")) + .collect(); + outputs.push(format!( + "ROW_NUMBER() OVER (PARTITION BY p.c0 ORDER BY p.c1) AS c{}", + column_count - 1 + )); + ctes.push(format!( + "step_{step} AS (SELECT {} FROM step_{} AS p)", + outputs.join(", "), + step - 1 + )); + } + format!( + "WITH {} SELECT * FROM step_{}", + ctes.join(",\n"), + cte_count - 1 + ) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_rules.rs b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_rules.rs new file mode 100644 index 0000000000..ba5c068406 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_rules.rs @@ -0,0 +1,405 @@ +use serde_json::Value; + +use crate::sql_quality::tests::helpers::{anchors, fixed, lint}; +use crate::sql_quality::tests::test_types::{QualityRemediationTestCase, QualityRuleTestCase}; + +#[test] +fn given_quality_rule_inputs_when_linting_then_findings_and_fixes_match() -> Result<(), String> { + let test_cases = [ + QualityRuleTestCase { + description: "long literal is reported without a split fix", + sql: "SELECT 'pending orders awaiting shipment' AS label", + rule: "SQBRSQL044", + relation_keys: "{}", + expected_anchors: &["'pending orders awaiting shipment'"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "short literal passes", + sql: "SELECT 'pending' AS label", + rule: "SQBRSQL044", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "typed literal is reported without a fix", + sql: "SELECT TIMESTAMP '2026-04-01 10:00:00.000000000' AS ordered_at", + rule: "SQBRSQL044", + relation_keys: "{}", + expected_anchors: &["'2026-04-01 10:00:00.000000000'"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ranking sort above the cap", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", + rule: "SQBRSQL043", + relation_keys: "{}", + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ranking sort within the cap", + sql: "SELECT RANK() OVER (ORDER BY o.a, o.order_id) AS rn FROM orders AS o", + rule: "SQBRSQL043", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ordering covers a declared key", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at, o.order_id) AS rn FROM orders AS o", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ordering without a key is unproven", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.ordered_at) AS rn FROM orders AS o", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "key proven through a CTE and a one-to-one join", + sql: "WITH lines AS (SELECT l.line_id, l.order_id FROM order_lines AS l), final AS (SELECT l.line_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY l.line_id) AS rn FROM lines AS l INNER JOIN orders AS o ON l.order_id = o.order_id) SELECT f.line_id, f.customer_id, f.rn FROM final AS f", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "key lost through a fan-out join", + sql: "WITH final AS (SELECT o.order_id, ROW_NUMBER() OVER (ORDER BY o.order_id) AS rn FROM orders AS o INNER JOIN order_lines AS l ON l.order_id = o.order_id) SELECT f.order_id, f.rn FROM final AS f", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]], "order_lines": [["line_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "GROUP BY grain is a key", + sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY o.customer_id), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", + rule: "SQBRSQL018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "positional GROUP BY grain is a key", + sql: "WITH totals AS (SELECT o.customer_id, SUM(o.amount) AS amount FROM orders AS o GROUP BY 1), final AS (SELECT t.customer_id, ROW_NUMBER() OVER (ORDER BY t.amount, t.customer_id) AS rn FROM totals AS t) SELECT f.customer_id, f.rn FROM final AS f", + rule: "SQBRSQL018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a later rn = 1 filter makes the partition a key", + sql: "WITH ranked AS (SELECT o.order_id, o.customer_id, ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn FROM orders AS o), latest AS (SELECT r.customer_id, r.order_id FROM ranked AS r WHERE r.rn = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a QUALIFY window cannot prove itself", + sql: "SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount DESC) = 1", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &["ROW_NUMBER"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a QUALIFY partition is a key for later steps", + sql: "WITH latest AS (SELECT o.customer_id, o.order_id FROM orders AS o QUALIFY ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) = 1), final AS (SELECT l.customer_id, ROW_NUMBER() OVER (ORDER BY l.customer_id) AS position FROM latest AS l) SELECT f.customer_id, f.position FROM final AS f", + rule: "SQBRSQL018", + relation_keys: r#"{"orders": [["order_id"]]}"#, + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "RANK ties are deterministic", + sql: "SELECT RANK() OVER (ORDER BY o.ordered_at) AS rn FROM orders AS o", + rule: "SQBRSQL018", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "unused CTE output is removed", + sql: "WITH orders AS (SELECT o.order_id, o.status, o.note FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id, r.status FROM orders AS r) SELECT f.order_id, f.status FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "star pass-through into the final CTE keeps every column", + sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT * FROM orders) SELECT * FROM final", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "star pass-through follows the reader's own reads", + sql: "WITH orders AS (SELECT o.order_id, o.note FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id FROM raw_orders AS o), staged AS (SELECT * FROM orders), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "a same-named cast of a source column is not a lateral read", + sql: "WITH orders AS (SELECT o.order_id, CAST(note AS VARCHAR) AS note, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["CAST(note AS VARCHAR) AS note"], + expected_fixed_sql: Some( + "WITH orders AS (SELECT o.order_id, o.note || 'x' AS tagged FROM raw_orders AS o), final AS (SELECT r.order_id, r.tagged FROM orders AS r) SELECT f.order_id, f.tagged FROM final AS f", + ), + }, + QualityRuleTestCase { + description: "an interpolated expression in a reader may read any column", + sql: "WITH orders AS (SELECT o.order_id, o.status FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r WHERE __sqlbuild_audit_parameter_0__) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an interpolated output is never reported", + sql: "WITH orders AS (SELECT o.order_id, __SQB_LINT_0002 FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT CTE reports without a fix", + sql: "WITH orders AS (SELECT DISTINCT o.order_id, o.note FROM raw_orders AS o), final AS (SELECT r.order_id FROM orders AS r) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["o.note"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "columns read in later join and filter clauses are used", + sql: "WITH orders AS (SELECT o.order_id, o.customer_id, o.status FROM raw_orders AS o), final AS (SELECT c.name FROM orders AS r INNER JOIN customers AS c ON r.customer_id = c.customer_id WHERE r.status = 'open' QUALIFY ROW_NUMBER() OVER (PARTITION BY r.order_id ORDER BY c.name) = 1) SELECT f.name FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a column read only through a nested star is used", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g.*) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a whole-row reference reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id, HASH(g) AS row_hash FROM g) SELECT f.order_id, f.row_hash FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COUNT(DISTINCT row) reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT COUNT(DISTINCT g) AS n FROM g) SELECT f.n FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COLUMNS(*) reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT g.order_id FROM g WHERE COLUMNS(*) IS NOT NULL) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "COLUMNS with a pattern reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), final AS (SELECT MAX(COLUMNS('order.*')) FROM g) SELECT * FROM final", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT star pass-through reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT DISTINCT * FROM g), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a positional ORDER BY over a star pass-through reads every column", + sql: "WITH g AS (SELECT r.order_id, r.amount FROM raw_orders AS r), staged AS (SELECT * FROM g ORDER BY 2 LIMIT 1), final AS (SELECT s.order_id FROM staged AS s) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an output alias read in WHERE or GROUP BY is used", + sql: "WITH g AS (SELECT r.order_id, r.amount * 2 AS doubled, r.customer_id % 2 AS bucket FROM raw_orders AS r WHERE doubled > 1 GROUP BY r.order_id, doubled, bucket), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "GROUP BY ALL grouping column is reported without a fix", + sql: "WITH g AS (SELECT r.customer_id, r.status, SUM(r.amount) AS total FROM raw_orders AS r GROUP BY ALL), final AS (SELECT g.customer_id, g.total FROM g) SELECT f.customer_id, f.total FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["r.status"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "an aggregate in an ungrouped select is reported without a fix", + sql: "WITH g AS (SELECT 'all' AS label, COUNT(*) AS n FROM raw_orders AS r), final AS (SELECT g.label FROM g) SELECT f.label FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["COUNT(*) AS n"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a set-returning projection is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, UNNEST(r.tags) AS tag FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["UNNEST(r.tags) AS tag"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "ORDER BY ALL with LIMIT is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, r.customer_id FROM raw_orders AS r ORDER BY ALL LIMIT 1), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["r.customer_id"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "DISTINCT ON is reported without a fix", + sql: "WITH g AS (SELECT DISTINCT ON (r.customer_id) r.order_id, r.amount FROM raw_orders AS r ORDER BY r.customer_id, r.order_id), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["r.amount"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a volatile projection is reported without a fix", + sql: "WITH g AS (SELECT r.order_id, RANDOM() AS draw FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["RANDOM() AS draw"], + expected_fixed_sql: None, + }, + QualityRuleTestCase { + description: "a deterministic scalar projection is removed", + sql: "WITH g AS (SELECT r.order_id, COALESCE(UPPER(r.status), 'none') AS label FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + rule: "SQBRSQL042", + relation_keys: "{}", + expected_anchors: &["COALESCE(UPPER(r.status), 'none') AS label"], + expected_fixed_sql: Some( + "WITH g AS (SELECT r.order_id FROM raw_orders AS r), final AS (SELECT g.order_id FROM g) SELECT f.order_id FROM final AS f", + ), + }, + ]; + for test_case in &test_cases { + let diagnostics: Vec = lint(test_case)?; + assert_eq!( + anchors(test_case.sql, &diagnostics), + test_case.expected_anchors, + "{}", + test_case.description + ); + assert_eq!( + fixed(test_case.sql, &diagnostics).as_deref(), + test_case.expected_fixed_sql, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_threshold_findings_when_linting_then_remediation_shows_the_exact_setting() +-> Result<(), String> { + let test_cases = [ + QualityRemediationTestCase { + description: "long literal points at a constant and the length setting", + sql: "SELECT 'pending orders awaiting shipment' AS label", + rule: "SQBRSQL044", + expected_fragments: &[ + "reference it with `@const`", + "(1) Only this model: add `constants (_status_pattern '')`", + "`_sqlbuild/_constants/` of the nearest folder", + "(3) Use top-level `constants/` only", + "the current value is 24", + "[rules.thresholds]\n max_literal_length = 32", + ], + }, + QualityRemediationTestCase { + description: "long ranking sort points at a row key and the sort setting", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a, o.b, o.order_id) AS rn FROM orders AS o", + rule: "SQBRSQL043", + expected_fragments: &[ + "unique row key as the final tie-breaker", + "HASH(source_partition, source_offset, line_index) AS row_key", + "the current value is 2", + "[rules.thresholds]\n max_ranking_order_by = 3", + ], + }, + QualityRemediationTestCase { + description: "unproven ranking points at upstream keys and a built row key", + sql: "SELECT ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.a) AS rn FROM orders AS o", + rule: "SQBRSQL018", + expected_fragments: &[ + "a `unique` column audit on a column that is also non-null", + "the `unique_key` of an incremental `merge` model", + "HASH(source_partition, source_offset, line_index) AS row_key", + ], + }, + ]; + for test_case in &test_cases { + let diagnostics: Vec = lint(&QualityRuleTestCase { + description: test_case.description, + sql: test_case.sql, + rule: test_case.rule, + relation_keys: "{}", + expected_anchors: &[], + expected_fixed_sql: None, + })?; + let remediation: &str = diagnostics + .first() + .and_then(|diagnostic| diagnostic["remediation"].as_str()) + .ok_or_else(|| format!("{}: no finding", test_case.description))?; + let present: Vec<&str> = test_case + .expected_fragments + .iter() + .copied() + .filter(|fragment| remediation.contains(fragment)) + .collect(); + assert_eq!( + present, test_case.expected_fragments, + "{}: {remediation}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_scale.rs b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_scale.rs new file mode 100644 index 0000000000..b1bf2179f6 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_quality_scale.rs @@ -0,0 +1,43 @@ +use std::time::Instant; + +use serde_json::{Value, json}; + +use crate::sql_lint::main::engine::lint_json; +use crate::sql_quality::tests::helpers::wide_cte_chain; +use crate::sql_quality::tests::test_types::QualityScaleTestCase; + +#[test] +fn given_many_wide_ctes_with_findings_when_linting_then_quality_rules_stay_linear() +-> Result<(), String> { + let test_cases = [QualityScaleTestCase { + description: "1,500 wide CTEs, each dropping a column and ranking without a key", + cte_count: 1_500, + column_count: 24, + expected_findings: 2_998, + expected_max_seconds: 20.0, + }]; + for test_case in &test_cases { + let request: Value = json!({ + "version": 1, + "sql": wide_cte_chain(test_case.cte_count, test_case.column_count), + "dialect": "duckdb", + "enabled_rules": ["SQBRSQL018", "SQBRSQL042", "SQBRSQL043", "SQBRSQL044"], + }); + let started: Instant = Instant::now(); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + assert_eq!( + response["diagnostics"].as_array().map_or(0, Vec::len), + test_case.expected_findings, + "{}", + test_case.description + ); + assert!( + elapsed <= test_case.expected_max_seconds, + "{}: {elapsed:.2}s", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/tests/test_types.rs b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_types.rs new file mode 100644 index 0000000000..d37c791c18 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/tests/test_types.rs @@ -0,0 +1,23 @@ +pub(crate) struct QualityRuleTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) rule: &'static str, + pub(crate) relation_keys: &'static str, + pub(crate) expected_anchors: &'static [&'static str], + pub(crate) expected_fixed_sql: Option<&'static str>, +} + +pub(crate) struct QualityScaleTestCase { + pub(crate) description: &'static str, + pub(crate) cte_count: usize, + pub(crate) column_count: usize, + pub(crate) expected_findings: usize, + pub(crate) expected_max_seconds: f64, +} + +pub(crate) struct QualityRemediationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) rule: &'static str, + pub(crate) expected_fragments: &'static [&'static str], +} diff --git a/crates/sqlbuild-rules-native/src/sql_quality/types.rs b/crates/sqlbuild-rules-native/src/sql_quality/types.rs new file mode 100644 index 0000000000..717e03a5ee --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_quality/types.rs @@ -0,0 +1,6 @@ +use std::collections::BTreeSet; + +/// Column sets that are unique for one relation, by lower-cased output column name. +pub(crate) type Keys = Vec>; +/// A unique column set over a select's joined input, as `(source alias, column)` pairs. +pub(crate) type InputKey = BTreeSet<(String, String)>; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs b/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs index 8a2cb3012d..ebe02b15a0 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-rules-native/src/sql_tokens/constants.rs @@ -1,5 +1,7 @@ //! Token positions that decide which authored words may change letter case. +use polyglot_sql::tokens::TokenType; + /// Column aliases after `AS` may be any keyword in DuckDB and PostgreSQL. pub(crate) const ALIAS_KEYWORD: &str = "AS"; /// The token that turns a preceding name into a function call. @@ -15,3 +17,19 @@ pub(crate) const CALL_SYNTAX_FUNCTIONS: [&str; 6] = [ "CONVERT", "TRY_CONVERT", ]; + +pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ + TokenType::From, + TokenType::Where, + TokenType::Group, + TokenType::GroupBy, + TokenType::Having, + TokenType::Qualify, + TokenType::Order, + TokenType::OrderBy, + TokenType::Limit, + TokenType::Window, + TokenType::Union, + TokenType::Except, + TokenType::Intersect, +]; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs index f7a6b602b6..7690ea8d16 100644 --- a/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod canonical_tokens; pub(crate) mod case_folding; pub(crate) mod is_unquoted_word; pub(crate) mod name_positions; +pub(crate) mod projection_spans; pub(crate) mod query_fingerprint; pub(crate) mod token_texts; diff --git a/crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs b/crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs new file mode 100644 index 0000000000..9a261b47b5 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_tokens/main/projection_spans.rs @@ -0,0 +1,41 @@ +//! Authored spans of the top-level items of one select list. + +use polyglot_sql::tokens::{Token, TokenType}; + +use crate::sql_tokens::constants::PROJECTION_ENDS; + +/// Spans of each top-level projection item of the select starting at token `select`. +pub(crate) fn projection_spans(tokens: &[Token], select: usize) -> Vec<(usize, usize)> { + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut depth: usize = 0; + let mut start: Option = None; + let mut end: usize = 0; + for token in tokens.iter().skip(select + 1) { + if matches!( + token.token_type, + TokenType::Space | TokenType::Break | TokenType::LineComment | TokenType::BlockComment + ) { + continue; + } + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen if depth == 0 => break, + TokenType::RParen => depth -= 1, + TokenType::Comma if depth == 0 => { + if let Some(first) = start.take() { + spans.push((first, end)); + } + continue; + } + kind if depth == 0 && PROJECTION_ENDS.contains(&kind) => break, + TokenType::Distinct | TokenType::All if depth == 0 && start.is_none() => continue, + _ => {} + } + start.get_or_insert(token.span.start); + end = token.span.end; + } + if let Some(first) = start { + spans.push((first, end)); + } + spans +} diff --git a/scripts/setting_help/__init__.py b/scripts/setting_help/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/setting_help/_helpers/__init__.py b/scripts/setting_help/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/setting_help/_helpers/scan.py b/scripts/setting_help/_helpers/scan.py new file mode 100644 index 0000000000..1456e09689 --- /dev/null +++ b/scripts/setting_help/_helpers/scan.py @@ -0,0 +1,55 @@ +"""Source scanning for setting names in authored diagnostic text.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +from scripts.setting_help.constants import ( + DOTTED_SETTING_INSTRUCTION_PATTERN, + RUST_STRING_DELIMITER, + RUST_TEST_PATH_MARKER, + YAML_FILE_TEXT_MODULES, + YAML_STYLE_SETTING_PATTERN, +) + + +def python_findings(repository_root: Path) -> list[str]: + findings: list[str] = [] + for path in sorted((repository_root / "src" / "sqlbuild").rglob("*.py")): + relative: str = path.relative_to(repository_root).as_posix() + if relative in YAML_FILE_TEXT_MODULES: + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + if isinstance(node, ast.Constant | ast.JoinedStr): + text: str = _literal_text(node) + if names_setting_without_snippet(text): + findings.append(f"{relative}:{node.lineno}: {text}") + return findings + + +def rust_findings(repository_root: Path) -> list[str]: + findings: list[str] = [] + for path in sorted((repository_root / "crates").rglob("src/**/*.rs")): + relative: str = path.relative_to(repository_root).as_posix() + if RUST_TEST_PATH_MARKER in relative: + continue + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if RUST_STRING_DELIMITER in line and names_setting_without_snippet(line): + findings.append(f"{relative}:{number}: {line.strip()}") + return findings + + +def names_setting_without_snippet(text: str) -> bool: + return bool( + YAML_STYLE_SETTING_PATTERN.search(text) or DOTTED_SETTING_INSTRUCTION_PATTERN.search(text) + ) + + +def _literal_text(node: ast.Constant | ast.JoinedStr) -> str: + parts: list[ast.expr] = list(node.values) if isinstance(node, ast.JoinedStr) else [node] + return "".join( + part.value + for part in parts + if isinstance(part, ast.Constant) and isinstance(part.value, str) + ) diff --git a/scripts/setting_help/constants.py b/scripts/setting_help/constants.py new file mode 100644 index 0000000000..6af1b4d7c8 --- /dev/null +++ b/scripts/setting_help/constants.py @@ -0,0 +1,25 @@ +"""Patterns for diagnostic text that names a setting without its exact TOML snippet.""" + +from __future__ import annotations + +import re + +SNIPPET_PATTERN: re.Pattern[str] = re.compile( + r"^ +\[([a-z_.]+)\]\n +([a-z_]+) = (.+)$", re.MULTILINE +) +YAML_STYLE_SETTING_PATTERN: re.Pattern[str] = re.compile( + r"`?\b(?:settings\.)?[a-z_]+:\s*(?:true|false|staged)\b|`[a-z_]+: [a-z_]+`" + r"|\[(?:dbt|settings|references|rules|janitor|defaults)\]\.[a-z_]+" +) +DOTTED_SETTING_INSTRUCTION_PATTERN: re.Pattern[str] = re.compile( + r"\b(?:[Ss]et|[Ee]nable|[Dd]isable|[Aa]dd|requires|allow)\b[^.;\n\"]{0,60}?" + r"\b(?:settings|references|janitor|dbt|rules|connection|defaults)\.[a-z_]+" +) +YAML_FILE_TEXT_MODULES: frozenset[str] = frozenset( + { + "src/sqlbuild/cli/commands/_helpers/playground/copy.py", + "src/sqlbuild/compiler/discovery/_helpers/yml/sources.py", + } +) +RUST_TEST_PATH_MARKER: str = "/tests/" +RUST_STRING_DELIMITER: str = '"' diff --git a/scripts/setting_help/main/__init__.py b/scripts/setting_help/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/setting_help/main/find_settings_without_snippet.py b/scripts/setting_help/main/find_settings_without_snippet.py new file mode 100644 index 0000000000..66004c08d0 --- /dev/null +++ b/scripts/setting_help/main/find_settings_without_snippet.py @@ -0,0 +1,13 @@ +"""Find authored diagnostic text that names a setting without its exact TOML snippet.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.setting_help._helpers.scan import python_findings, rust_findings + + +def find_settings_without_snippet(repository_root: Path) -> list[str]: + """Return `path:line: text` for Python and Rust strings that name a setting loosely.""" + + return [*python_findings(repository_root), *rust_findings(repository_root)] diff --git a/scripts/setting_help/main/parse_setting_snippet.py b/scripts/setting_help/main/parse_setting_snippet.py new file mode 100644 index 0000000000..1924fb6504 --- /dev/null +++ b/scripts/setting_help/main/parse_setting_snippet.py @@ -0,0 +1,18 @@ +"""Parse the TOML snippet shown by a diagnostic help text.""" + +from __future__ import annotations + +import re +import tomllib + +from scripts.setting_help.constants import SNIPPET_PATTERN + + +def parse_setting_snippet(help_text: str) -> tuple[str, str, object] | None: + """Return `(section, key, value)` from the first `[section]` / `key = value` snippet.""" + + match: re.Match[str] | None = SNIPPET_PATTERN.search(help_text) + if match is None: + return None + section, key, value = match.groups() + return section, key, tomllib.loads(f"value = {value}")["value"] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index e00d270193..8af247c1a8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -108,12 +108,21 @@ that are known to preserve meaning, then formats the layout. `--fix --check` rep changes without writing files, and `--fix --diff` prints their diff. Edits are planned in memory and repeated until no more apply; overlapping edits wait for the next -pass. Every pass recompiles the project and compares output columns (including types and -nullability), dependencies, and column lineage of each edited model and everything downstream of it -against the original compilation. When a pass cannot be proved, the edits of each file are checked -on their own: files whose edits cannot be proved keep their original contents and report -`format-fix-verification-failed`, while the other files are still fixed. A cycle or the 128-pass -limit fails without writing any edit. +pass. Every pass recompiles the project and compares it with the original compilation, so a +project that already has compile errors can still be fixed: + +- An edited file may lose diagnostics but must not gain any. Diagnostics are compared by code, + resource and message, not by line. +- Each edited model must keep its output columns, types and nullability, dependencies, and column + lineage. Lineage that is unavailable both before and after counts as unchanged when the columns + match. +- No other file's diagnostics or model outputs may change. + +A failure is attributed to the edited file that caused it, and that file keeps its original +contents and reports `format-fix-verification-failed` with the reason, while the other files are +still fixed. When a difference cannot be attributed, each file's edits are checked on their own. A +file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the +reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. | Rule | Decision | Reason | | --- | --- | --- | @@ -128,6 +137,7 @@ limit fails without writing any edit. | SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | | SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | | SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | +| SQBRSQL042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | Edits that touch macro or intrinsic expansions are refused, and so are edits outside model SQL bodies, because recompiling cannot prove their output contract. Findings without a proposed edit diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 02affca56e..f48620c5a6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -19,6 +19,7 @@ Online: https://sqlbuild.com/docs/concepts/audits/ - Seed audits - Severity - Run scope +- Skipping SQL analysis - Running audits standalone Audits are SQL queries that verify data quality. Violation audits return invalid rows. Measurement @@ -425,6 +426,15 @@ The `4/4` indicates the audit passed for all 4 microbatch batches. If a model is not incremental, `delta_and_final` degrades to `final` automatically. +## Skipping SQL analysis + +Compile checks audit SQL like model SQL: unknown columns and functions, type mismatches and syntax +errors are compile errors. For SQL the parser cannot read, add `sql_analysis false` to the `AUDIT` +header of a singular or generic audit; the audit still runs. Audits attached to a model with +`sql_analysis false`, and every audit in a `--no-sql-analysis` run, are not analysed. With +`[settings] require_sql_analysis = true`, the opt-out is accepted only on audit SQL that cannot be +parsed (see [Requiring SQL analysis](models/configuration.md#requiring-sql-analysis)). + ## Running audits standalone ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 66c5d81e63..e7d71af19c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -65,6 +65,45 @@ A model cannot re-enable analysis when either broader gate disables it. The olde and `--no-sql-validation` spellings remain compatibility aliases; use `sql_analysis` for new configuration. +### Requiring SQL analysis + +A project can stop opt-outs from hiding real findings: + +```toml +# sqlbuild_project.toml +[settings] +require_sql_analysis = true # default false +``` + +With this setting, `sql_analysis false` from a MODEL header or a `[path_defaults]` entry is accepted +only on a model whose SQL the parser cannot read: its query or one of its SQL hooks. On a model that +parses, compile reports a `P009` +error at the header key or `[path_defaults]` entry, with the findings the opt-out was hiding counted +by kind: + +```text +error[P009]: `sql_analysis false` is not needed for model 'orders_summary' + --> models/marts/orders_summary.sql:4:3 + = note: sqlbuild_project.toml sets [settings] require_sql_analysis = true, which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; this model parses successfully. + = help: remove `sql_analysis false` and fix the findings it was hiding: 2 unknown columns, 1 type mismatch (run `sqb compile` to see them) + = help: to allow `sql_analysis false` on any model, SQL test or audit, set this in sqlbuild_project.toml: + [settings] + require_sql_analysis = false +``` + +`TEST(...)` and `AUDIT(...)` headers (singular and generic audits) also accept +`sql_analysis false`, which skips compile-time SQL checks on that test or audit; the SQL still runs. +SQL tests of a model with `sql_analysis false`, and audits attached to it, are not analysed either. +`require_sql_analysis = true` applies the same rule to these headers: the opt-out is accepted only +when that test or audit SQL cannot be parsed, otherwise compile reports `P009` at the header key. + +`[settings] sql_analysis = false` in `sqlbuild_local.toml` is then a `D001` error, and +`require_sql_analysis` itself can only be set in `sqlbuild_project.toml`. `--no-sql-analysis` still +turns analysis off for one run, including the checks on audits, SQL tests and SQL hooks. + +When the parser cannot read a model, the `P001` syntax error shows the exact MODEL header entry +(`sql_analysis false,`) and asks you to report the SQL so the parser can support it. + ## Table fields | Field | Description | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 2b0e640a03..a772fdacb5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -413,6 +413,7 @@ Global feature toggles: ```toml [settings] sql_analysis = true +require_sql_analysis = false query_change_tracking = true column_contract_mode = "implicit" concurrency = 1 @@ -426,6 +427,7 @@ default_audit_run_scope = "final" | Field | Default | Description | |-------|---------|-------------| | `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | +| `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | | `concurrency` | `1` | Maximum parallel model execution (currently serial only) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 19424f3424..833d151170 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -6,6 +6,14 @@ Online: https://sqlbuild.com/docs/concepts/rules/configuration-and-selection/ +## Contents + +- Codes and families +- Rule options +- Focused selection +- Join keys and final CTE names +- SQL quality limits and ranking determinism + Configure built-in and custom Rules together: ```toml @@ -102,6 +110,71 @@ owns the terminal `SELECT`'s plain projection from that last CTE, so select both the `SQBRSQL` family, for the complete convention. Parentheses around the entire query do not create a nested CTE scope. +## SQL quality limits and ranking determinism + +These Rules are errors with project-wide limits. No model can opt out of a limit locally. + +```toml +# sqlbuild_project.toml +[rules.thresholds] +max_literal_length = 100 # default +max_ranking_order_by = 5 # default +``` + +- `SQBRSQL042`: every output column of a non-final CTE must be read later in the same model, + in any clause of a later CTE or the final query, including through a later `SELECT *` + pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQBRSQL005`), + CTEs with a column list, recursive queries and SQL-test fixtures are exempt. A later step that + uses an `@` interpolation in an expression may read any column, so it keeps every column, and + interpolated outputs are never reported. A later step that reads the CTE as a whole row (the + alias itself, as in `HASH(g)` or `COUNT(DISTINCT g)`), through a star nested in an expression + (`HASH(g.*)`), through `COLUMNS(...)`, or through `SELECT DISTINCT *` or a grouped `SELECT *` + reads every column. An output alias reused in the CTE's own `WHERE`, `GROUP BY`, `HAVING`, + `QUALIFY` or `ORDER BY` is read. A `GROUP BY` key can be dropped from the output while it stays + in `GROUP BY`. + `sqb format --fix` removes an unused column only when the removal keeps every row: the column + must be a column reference or a deterministic scalar expression (no aggregate, window, + set-returning or table function, subquery, or volatile function), and the CTE must not use + `SELECT DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an aggregate without + `GROUP BY`, or positional `GROUP BY`/`ORDER BY`. Other findings are reported without a fix and + name the reason. The fix repeats until nothing changes. +- `SQBRSQL043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may + sort by at most `max_ranking_order_by` expressions. Order by the one or two columns that decide + which row wins, then a unique row key as the final tie-breaker. If the rows have no unique + column, build one in the upstream model from the columns that identify a row (for example + `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` + audit, so `SQBRSQL018` can prove the order. Raise the project limit only when the order + genuinely needs more keys. +- `SQBRSQL044`: a string literal longer than `max_literal_length` characters is an error. Define + the value once as a [constant](../constants.md) in the narrowest + [scope](../declaration-scopes.md) that covers every use, and reference it with + `@const`: a model-private `constants (_status_pattern '...')` entry in the `MODEL` header when + only that model uses it; a `CONSTANT (name status_pattern, value '...');` file in + `_sqlbuild/_constants/` (that folder only) or `_sqlbuild/constants/` (that folder and below) of + the nearest folder holding every use; top-level `constants/` only for project-wide values. + Otherwise shorten the literal. When the identical literal is reported in other files, the help + lists them (up to five) and names the nearest common folder and its constants directory. + Literals produced by `@const` or a macro are not reported, and there is no automatic fix. +- `SQBRSQL018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven + deterministic: `PARTITION BY` and `ORDER BY` together must cover a unique key of the ranked rows. + Keys come from `unique` column audits and model-level `unique` audits naming a column, when the + column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of + incremental `merge` models and of `delete_insert` models matched by key without a cursor. + Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s do not prove one + row per key. Keys are followed through projections, renames, filters, one-to-one joins, + `GROUP BY` grains (named or positional), `SELECT DISTINCT`, `UNION`, and + `ROW_NUMBER() ... = 1` deduplication in `QUALIFY` or a later `WHERE`; a `QUALIFY` key holds for + later steps, never for the window that defines it. A join that can match a row more than once, + `UNION ALL`, and relations without a declared key lose the proof, and "cannot prove" is an + error. To comply, declare the key upstream (for example + `columns (order_id (audits [unique, not_null]))`), or end the order with a unique + tie-breaker, building a row key upstream when none exists. `RANK` and + `DENSE_RANK` give tied rows the same value, so they need no proof. + +A finding that `sqb format --fix` can fix says so in text output and carries `"fixable": true` in +`sqb rules --json run` output; `sqb compile` adds the same note to the diagnostic. + Family selectors such as `select = ["SQBRSQL", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules -on upgrade, so a release that adds `SQBRSQL040` and `SQBRSQL041` can expose computed join keys or -differently named final CTEs that were accepted before. +on upgrade, so a release that adds `SQBRSQL040` to `SQBRSQL044` can expose computed join keys, +differently named final CTEs, unused CTE columns, long literals or ranking sorts that were accepted +before. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md index d9525d4ab7..75ecd8dff0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -51,6 +51,18 @@ Known output shapes also validate `unique_key`, `cursor`, `cursor_type`, `cursor `config.partition_column`, `row_diff_exclude_columns`, `row_diff_tolerances.by_column`, source `cursor_column`, audit expressions/relationships/accepted values, and SQL-test mock/expected names. +The same binding, function-name and type checks run on every SQL body a project owns, not only +model queries: singular and generic audits (including measurement and evidence queries), SQL unit +tests (mock, helper, expected and assertion CTEs), and inline and named SQL hooks. An audit reading +`__ref("orders")` is checked against the closed `orders` output shape, so `WHERE o.shipped_at IS +NULL` on a model without that column fails compile instead of the warehouse. Each diagnostic names +its owner: `SQL test 'orders_are_open': ...`, `audit 'recent_rows' on model 'orders': ...` (with the +attached column when there is one), or `post_hooks[1] SQL hook: ...` at the hook file. Type +mismatches show the explicit conversion to write, such as `CAST('1' AS NUMBER)` in a test's expected +rows or `TO_DATE(@date_column)` in a generic audit. An +audit or SQL-test body the SQL parser cannot read is a P001 syntax error, never a silent pass. +Relations without a closed shape, such as physical tables written in hooks, stay open. + Expression type checks are enabled for **DuckDB (including MotherDuck), PostgreSQL, Snowflake, and BigQuery**. Other adapters retain reference and semantic checks without expression type checks. @@ -95,6 +107,9 @@ diagnostic warning count. Escape hatches are `MODEL (sql_analysis false)`, the corresponding path default, and `--no-sql-analysis`. Disabling SQL analysis also disables inference and lineage for that model. +`[settings] require_sql_analysis = true` limits the MODEL and path-default opt-outs to SQL the +parser cannot read; see [Requiring SQL analysis](models/configuration.md#requiring-sql-analysis). +Help that names a setting always shows the exact file, section and `key = value` to write. ## Reading diagnostics @@ -202,3 +217,6 @@ for example `order_id = CAST(customer_code AS INTEGER)`, or an explicit comparis B002–B005 predate these changes and overlap the executor's separately phased build codes. New semantic codes use distinct ranges. Authored model diagnostics retain source paths and positions. +Cast, coercible-literal and set-operation findings point at the offending cast, literal or branch +projection, and every finding is reported: two failing casts in one query are two diagnostics at +their own locations, never one merged diagnostic at the statement start. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index f49af3a666..e91ebf9a39 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -24,6 +24,7 @@ Online: https://sqlbuild.com/docs/concepts/testing/ - Multiple tests per file - Repeating test logic - Test placement +- Skipping SQL analysis - Running tests SQLBuild supports SQL-native unit tests that validate model logic by comparing actual query results against expected values. Tests can chain across multiple models (multi-model tests), use macros for reusable mock data, and include zero-row assertions. @@ -688,6 +689,14 @@ tests/ ... ``` +## Skipping SQL analysis + +Compile checks SQL test fixtures and assertions like model SQL. For SQL the parser cannot read, +add `sql_analysis false` to the `TEST` header; the test still runs. Tests of a model with +`sql_analysis false`, and every test in a `--no-sql-analysis` run, are not analysed. With +`[settings] require_sql_analysis = true`, the opt-out is accepted only on test SQL that cannot be +parsed (see [Requiring SQL analysis](models/configuration.md#requiring-sql-analysis)). + ## Running tests Tests run automatically during `sqb build` in DAG order, before their target model is materialized. diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index dd2a4034e9..b36671c154 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1375,7 +1375,12 @@ def connect(self, config: dict[str, Any]) -> _BigQueryConnection: raise AdapterUserError( message="BigQuery connection requires non-empty 'project'", code="A101", - help="set connection.project in sqlbuild_local.toml or the active target", + help=( + "set the Google Cloud project in sqlbuild_local.toml (or in the active " + "target's connection):\n" + " [connection]\n" + ' project = "my-project"' + ), ) try: diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 5654563290..d6f0534ceb 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -52,6 +52,7 @@ from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.rule_engine.classes.early_sql_lint import EarlySqlLint +from sqlbuild.rule_engine.constants import RULE_FIX_AVAILABLE_NOTE from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress from sqlbuild.rule_engine.main.run_rules import run_rules @@ -233,6 +234,7 @@ def _analyze_compile_project( line=fault.line, column=fault.column, help=fault.remediation, + notes=(RULE_FIX_AVAILABLE_NOTE,) if fault.fixable else (), affected_rules=fault.affected_rules, ) for fault in rules_result.findings diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py index d20982cd5d..3a234a60b0 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/runs.py @@ -5,6 +5,7 @@ import json import tomllib from pathlib import Path +from typing import cast from sqlbuild.cli.commands._helpers.lint.diagnostics import format_lint_diagnostics from sqlbuild.compiler.discovery.constants import LOCAL_CONFIG_FILENAME @@ -13,9 +14,13 @@ ADAPTER_DIALECT_TRANSLATIONS, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, + DEFAULT_MAX_LITERAL_LENGTH, + DEFAULT_MAX_RANKING_ORDER_BY, FORMAT_SECTION_KEY, LINE_WIDTH_KEY, MAX_DESCRIPTION_LINES_KEY, + MAX_LITERAL_LENGTH_THRESHOLD, + MAX_RANKING_ORDER_BY_THRESHOLD, PROJECT_CONFIG_FILENAME_KEY, ) from sqlbuild.lint.models import ( @@ -31,10 +36,12 @@ def resolve_lint_config(*, project_dir: Path) -> LintConfig: max_description_lines: int = DEFAULT_MAX_DESCRIPTION_LINES line_width: int = DEFAULT_LINE_WIDTH dialect: str = "generic" + thresholds: dict[str, object] = {} config_file: Path = project_dir / PROJECT_CONFIG_FILENAME_KEY if config_file.is_file(): with config_file.open("rb") as handle: payload: dict[str, object] = tomllib.load(handle) + thresholds = _rule_thresholds(payload) format_section: object = payload.get(FORMAT_SECTION_KEY) raw_adapter: object = payload.get(ADAPTER_CONFIG_KEY) if isinstance(raw_adapter, str): @@ -61,6 +68,31 @@ def resolve_lint_config(*, project_dir: Path) -> LintConfig: line_width=line_width, max_description_lines=max_description_lines, dialect=dialect, + max_literal_length=_threshold( + thresholds=thresholds, + key=MAX_LITERAL_LENGTH_THRESHOLD, + default=DEFAULT_MAX_LITERAL_LENGTH, + ), + max_ranking_order_by=_threshold( + thresholds=thresholds, + key=MAX_RANKING_ORDER_BY_THRESHOLD, + default=DEFAULT_MAX_RANKING_ORDER_BY, + ), + ) + + +def _rule_thresholds(payload: dict[str, object]) -> dict[str, object]: + rules: object = payload.get("rules") + if not isinstance(rules, dict): + return {} + thresholds: object = cast(dict[str, object], rules).get("thresholds") + return cast(dict[str, object], thresholds) if isinstance(thresholds, dict) else {} + + +def _threshold(*, thresholds: dict[str, object], key: str, default: int) -> int: + value: object = thresholds.get(key) + return ( + value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else default ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/connection.py b/src/sqlbuild/cli/commands/_helpers/runtime/connection.py index fc4a2103ee..8a8a9e20f0 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/connection.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/connection.py @@ -87,7 +87,7 @@ def _warn_if_duckdb_has_snowflake_like_keys( print( "Warning: DuckDB adapter is active, but connection contains " f"Snowflake-like keys: {key_list}. If this is a Snowflake local config, " - "add top-level `adapter: snowflake` to sqlbuild_local.toml.", + 'add `adapter = "snowflake"` at the top of sqlbuild_local.toml, before any section.', file=sys.stderr, ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index 44b4c2be3b..d05ac945ec 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -27,7 +27,6 @@ from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios from sqlbuild.cli.commands.constants import ( SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - SQL_ANALYSIS_CONFIG_KEY, ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( @@ -39,6 +38,7 @@ from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline from sqlbuild.compiler.pipeline.models import ( @@ -69,11 +69,9 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: force: bool = limit_inputs.force if no_sql_validation: raise CliUserError( - "scenario capture requires SQL analysis", + "scenario capture requires SQL analysis, but `--no-sql-analysis` turns it off", code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help=( - "Enable settings.sql_analysis when capturing snapshots for local scenario replay." - ), + help="run the capture without `--no-sql-analysis`", ) effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() @@ -170,23 +168,14 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: def _validate_capture_sql_analysis_enabled(*, discovered_inputs: DiscoveredProjectInputs) -> None: - if not _effective_sql_analysis_and_validation_enabled(discovered_inputs=discovered_inputs): + guidance: tuple[str, str] | None = sql_analysis_off_guidance( + discovered_inputs=discovered_inputs, + purpose="to capture snapshots for local scenario replay", + ) + if guidance is not None: + note, help_text = guidance raise CliUserError( - "scenario capture requires SQL analysis", + f"scenario capture requires SQL analysis; {note}", code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help=( - "Enable settings.sql_analysis when capturing snapshots for local scenario replay." - ), + help=help_text, ) - - -def _effective_sql_analysis_and_validation_enabled( - *, discovered_inputs: DiscoveredProjectInputs -) -> bool: - setting_overrides: frozenset[str] = discovered_inputs.local_config.setting_overrides - sql_analysis_enabled: bool = ( - discovered_inputs.local_config.settings.sql_analysis - if SQL_ANALYSIS_CONFIG_KEY in setting_overrides - else discovered_inputs.project_config.settings.sql_analysis - ) - return sql_analysis_enabled diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index e15a98f620..6143b6b847 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -36,7 +36,6 @@ SCENARIO_CLI_LOCAL_RETAIN_UNSUPPORTED, SCENARIO_CLI_LOCAL_SNAPSHOT_FLAG_REQUIRED, SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - SQL_ANALYSIS_CONFIG_KEY, SUCCESS_STATUS, ) from sqlbuild.cli.commands.exceptions import CliUserError @@ -55,6 +54,7 @@ from sqlbuild.cli.progress.main._write_execution_header import write_execution_header from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compile_only import run_compile_only_pipeline from sqlbuild.compiler.pipeline.models import ( @@ -286,29 +286,23 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: def _validate_local_scenario_sql_analysis_enabled( *, discovered_inputs: DiscoveredProjectInputs, no_sql_validation: bool ) -> None: - if no_sql_validation or not _effective_sql_analysis_and_validation_enabled( - discovered_inputs=discovered_inputs - ): + if no_sql_validation: raise CliUserError( - "scenario test --local requires SQL analysis", + "scenario test --local requires SQL analysis, but `--no-sql-analysis` turns it off", code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help=( - "Enable settings.sql_analysis when running local scenario replay, " - "snapshot sync, or snapshot refresh." - ), + help="run the command without `--no-sql-analysis`", ) - - -def _effective_sql_analysis_and_validation_enabled( - *, discovered_inputs: DiscoveredProjectInputs -) -> bool: - setting_overrides: frozenset[str] = discovered_inputs.local_config.setting_overrides - sql_analysis_enabled: bool = ( - discovered_inputs.local_config.settings.sql_analysis - if SQL_ANALYSIS_CONFIG_KEY in setting_overrides - else discovered_inputs.project_config.settings.sql_analysis + guidance: tuple[str, str] | None = sql_analysis_off_guidance( + discovered_inputs=discovered_inputs, + purpose="to run local scenario replay, snapshot sync, or snapshot refresh", ) - return sql_analysis_enabled + if guidance is not None: + note, help_text = guidance + raise CliUserError( + f"scenario test --local requires SQL analysis; {note}", + code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, + help=help_text, + ) def _sync_local_snapshots( diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py index 7277f7950a..dbb1e9d2b0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py @@ -9,9 +9,18 @@ _replace_refs_with_stubs, substitute_placeholder_defaults, ) +from sqlbuild.compiler.compile.constants import SQL_ANALYSIS_OPT_OUT_ENTRY from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.discovery.constants import ( + PROJECT_CONFIG_FILENAME, + SETTINGS_SECTION, + SQL_ANALYSIS_CONFIG_KEY, +) from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.model_header_help import model_header_help +from sqlbuild.errors.setting_help.main.setting_help import setting_help def validate_sql_syntax( @@ -34,12 +43,8 @@ def validate_sql_syntax( if error_message is None: return raise CompileInputError( - f"SQL syntax error in model '{model_name}' ({file_path}): {error_message}\n\n" - f"To skip SQL analysis for this model, add `sql_analysis: false` " - f"to the MODEL header.\n" - f"To disable project-wide, set `settings.sql_analysis: false` " - f"in sqlbuild_project.toml.\n" - f"To skip for this run, use `--no-sql-analysis`." + f"SQL syntax error in model '{model_name}' ({file_path}): {error_message}", + help=sql_analysis_opt_out_help(model_owned=True), ) from None @@ -180,7 +185,9 @@ def _validate_sql_syntax_with_message( except polyglot_module.PolyglotError as error: error_message = str(error) if error_message is not None: - _raise_sql_validation_error(error_prefix=error_prefix, error_message=error_message) + _raise_sql_validation_error( + error_prefix=error_prefix, error_message=error_message, model_owned=False + ) def _validate_hook_sql_with_message( @@ -199,7 +206,9 @@ def _validate_hook_sql_with_message( ) error_message: str | None = _validate_sql_with_polyglot(sql=cleaned_sql, dialect=dialect) if error_message is not None: - _raise_sql_validation_error(error_prefix=error_prefix, error_message=error_message) + _raise_sql_validation_error( + error_prefix=error_prefix, error_message=error_message, model_owned=True + ) def _clean_sql_for_validation( @@ -217,12 +226,34 @@ def _clean_sql_for_validation( return cleaned_sql -def _raise_sql_validation_error(*, error_prefix: str, error_message: str) -> None: +def _raise_sql_validation_error( + *, error_prefix: str, error_message: str, model_owned: bool +) -> None: raise CompileInputError( - f"{error_prefix}: {error_message}\n\n" - f"To skip SQL analysis for this model, add `sql_analysis: false` " - f"to the MODEL header.\n" - f"To disable project-wide, set `settings.sql_analysis: false` " - f"in sqlbuild_project.toml.\n" - f"To skip for this run, use `--no-sql-analysis`." + f"{error_prefix}: {error_message}", help=sql_analysis_opt_out_help(model_owned=model_owned) ) from None + + +def sql_analysis_opt_out_help(*, model_owned: bool = True) -> str: + """Help for SQL the parser rejects: the exact header, project and single-run opt-outs.""" + + report: str = ( + "if this SQL is valid for your warehouse, please report it so the parser can support it" + ) + return join_helps( + model_header_help( + purpose="if this SQL is valid for your warehouse, skip SQL analysis for this model", + entry=SQL_ANALYSIS_OPT_OUT_ENTRY, + follow_up="and please report the SQL so the parser can support it", + ) + if model_owned + else report, + setting_help( + purpose="SQL analysis is on for this project; to turn it off for every model", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=SQL_ANALYSIS_CONFIG_KEY, + value=False, + ), + "to skip SQL analysis for one run, use `--no-sql-analysis`", + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py index f5682e2d82..00b756a42c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py @@ -21,6 +21,7 @@ known_function_names, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql import get_resource_sql_diagnostics from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledModel, @@ -60,8 +61,9 @@ def get_semantic_metadata_diagnostics( *, project: CompiledProject, profile: ExpressionInferenceProfile, + resource_sql_analysis: bool = True, ) -> tuple[CompilerDiagnostic, ...]: - """Reject metadata names only where absence can be proven.""" + """Reject metadata names only where absence can be proven; resource SQL only when analysed.""" if not project.settings.sql_analysis: return () @@ -175,6 +177,10 @@ def get_semantic_metadata_diagnostics( ) ) diagnostics.extend(_sql_test_errors(project=project, shapes=shapes, profile=profile)) + if resource_sql_analysis: + diagnostics.extend( + get_resource_sql_diagnostics(project=project, shapes=shapes, profile=profile) + ) return tuple(diagnostics) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 96bf3c844f..063daf4ac3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -459,6 +459,7 @@ def assemble_compiled_project( return complete_semantic_diagnostics( project=project, profile=profile, + resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, binding_results={ name: analysis.polyglot_analysis.binding_diagnostics for name, analysis in model_sql_analysis_by_name.items() @@ -617,6 +618,7 @@ def _assemble_compiled_model( ), binding_validated=(polyglot_analysis.binding_validated if sql_analysis_enabled else False), dynamic_column_contract=dynamic_column_contract, + rejected_sql_analysis_opt_out=model_input.rejected_sql_analysis_opt_out, ) @@ -1491,7 +1493,11 @@ def _binding_compiler_diagnostics( line, column, ) - if key in seen and diagnostic.severity == DiagnosticSeverity.ERROR: + if ( + key in seen + and diagnostic.start is not None + and diagnostic.severity == DiagnosticSeverity.ERROR + ): continue seen.add(key) result.append( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index f1961fc5b0..57ac4780ff 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -48,6 +48,9 @@ validate_preserved_logical_namespace, ) from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies +from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( + rejected_sql_analysis_opt_out, +) from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( merge_call_site_references, resource_references, @@ -117,7 +120,9 @@ ModelConfigScanCache, ModelHeaderColumnCache, ModelInputBuildContext, + SqlAnalysisOptOutRequest, ) +from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredHookFunction, @@ -158,6 +163,7 @@ SchemaModelEntry, SchemaSeedEntry, SettingsConfig, + SourceLocation, TargetConfig, ) @@ -232,6 +238,7 @@ def for_model( class _ModelValidationContext: effective_settings: SettingsConfig no_sql_validation: bool + project_config_path: Path defer_model_sql_validation: bool external_sql_reference_resolver: ExternalSqlReferenceResolver | None extract_references: Callable[[str], tuple[CompileSqlReference, ...]] @@ -434,6 +441,7 @@ def _build_model_inputs( validation_context: _ModelValidationContext = _ModelValidationContext( effective_settings=effective_settings, no_sql_validation=no_sql_validation, + project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, @@ -588,7 +596,7 @@ def _build_model_inputs( if isinstance(raw_placeholders, dict) else None ) - sql_validation_enabled, references = _validate_model_input( + sql_validation_enabled, references, rejected_opt_out = _validate_model_input( context=validation_context, model_file=model_file, config=effective_config, @@ -701,6 +709,7 @@ def _build_model_inputs( macro_source_sql=declaration_expanded_sql, references=references, sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, enum_declarations=tuple(declarations.local_enums.values()), constant_declarations=tuple(declarations.local_constants.values()), enum_columns=enum_columns, @@ -729,6 +738,7 @@ def _build_model_inputs( references=references, schema_entry=header_schema_entry, sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, enum_declarations=tuple(declarations.local_enums.values()), constant_declarations=tuple(declarations.local_constants.values()), enum_columns=enum_columns, @@ -783,13 +793,31 @@ def _validate_model_input( sql_validation_placeholders: dict[str, str] | None, model_schema_columns: tuple[SchemaColumn, ...] | None, argument_references: tuple[CompileSqlReference, ...], -) -> tuple[bool, tuple[CompileSqlReference, ...]]: +) -> tuple[bool, tuple[CompileSqlReference, ...], SourceLocation | None]: model_name: str = model_file.file_path.stem sql_validation_enabled: bool = _model_sql_validation_gate( effective_settings=context.effective_settings, no_sql_validation=context.no_sql_validation, model_config=config, ) + rejected_opt_out: SourceLocation | None = ( + None + if sql_validation_enabled + else rejected_sql_analysis_opt_out( + SqlAnalysisOptOutRequest( + model_file=model_file, + config=config, + settings=context.effective_settings, + no_sql_validation=context.no_sql_validation, + query_sql=cursor_intrinsics_analysis_sql( + sql=expanded_query_sql, cursor_type=config.values.get("cursor_type") + ), + placeholders=sql_validation_placeholders, + project_config_path=context.project_config_path, + ) + ) + ) + sql_validation_enabled = sql_validation_enabled or rejected_opt_out is not None if sql_validation_enabled and not context.defer_model_sql_validation: validate_sql_syntax( query_sql=cursor_intrinsics_analysis_sql( @@ -846,7 +874,7 @@ def _validate_model_input( query_sql=expanded_query_sql, custom_materialization_names=context.custom_materialization_names, ) - return sql_validation_enabled, references + return sql_validation_enabled, references, rejected_opt_out def _build_visible_declaration_indexes( diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index 53fa6c6612..8020d1a704 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -17,6 +17,7 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileModelConfig +from sqlbuild.compiler.discovery.main.microbatch_guidance import microbatch_guidance from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_CONFIG_KEY from sqlbuild.compiler.planner.types import ( ContractPolicy, @@ -796,9 +797,10 @@ def validate_microbatch_project_capability( and batch_concurrency > 1 and not settings.microbatch_concurrency ): + note, help_text = microbatch_guidance() raise CompileInputError( - f"model '{model_name}': batch_concurrency > 1 requires " - "settings.microbatch_concurrency = true" + f"model '{model_name}': batch_concurrency > 1 requires concurrent microbatches; {note}", + help=help_text, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py index fead671200..20fd94da22 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py @@ -170,7 +170,7 @@ def explain_diagnostics(project: CompiledProject) -> CompiledProject: diagnostic = replace( diagnostic, message=sentence_message(diagnostic.message), - help=semantic_help(diagnostic.code) or diagnostic.help, + help=diagnostic.help or semantic_help(diagnostic.code), ) diagnostics.append(diagnostic) return replace( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index 982b0bab7f..cb363cf8e0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -18,6 +18,9 @@ unaliased_output_columns, update_binding_models, ) +from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( + reject_unneeded_sql_analysis_opt_outs, +) from sqlbuild.compiler.compile._helpers.diagnostics.type_recovery import recover_output_types from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -35,6 +38,7 @@ def complete_semantic_diagnostics( project: CompiledProject, profile: ExpressionInferenceProfile, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + resource_sql_analysis: bool = True, ) -> CompiledProject: """Recover type facts before metadata checks, then explain retained root diagnostics.""" project = recover_output_types(project=project, binding_results=binding_results) @@ -42,10 +46,12 @@ def complete_semantic_diagnostics( project, diagnostics=( *project.diagnostics, - *get_semantic_metadata_diagnostics(project=project, profile=profile), + *get_semantic_metadata_diagnostics( + project=project, profile=profile, resource_sql_analysis=resource_sql_analysis + ), ), ) - return explain_diagnostics(recover_diagnostics(project)) + return reject_unneeded_sql_analysis_opt_outs(explain_diagnostics(recover_diagnostics(project))) def recover_diagnostics(project: CompiledProject) -> CompiledProject: diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py new file mode 100644 index 0000000000..a038120917 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py @@ -0,0 +1,523 @@ +"""Run the model compile-time SQL checks on audits, SQL tests and SQL hooks.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( + get_authored_binding_location, +) +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import binding_relation_names +from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql_help import resource_sql_help +from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( + unneeded_opt_out_diagnostic, +) +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import ( + CompiledAudit, + CompiledProject, + CompiledSqlExpansion, + CompilerDiagnostic, + CompileSqlReference, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY +from sqlbuild.compiler.sql_analysis.constants import BINDING_UNKNOWN_TABLE_INTERNAL_CODE +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from sqlbuild.compiler.sql_analysis.main._resolve_binding_column import resolve_binding_column +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.models import ( + SqlBindingDiagnostic, + SqlBindingResult, + SqlSchemaValidationRequest, +) +from sqlbuild.spec.contracts.models import SourceLocation + +_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +_QUOTED_NAME_PATTERN: re.Pattern[str] = re.compile(r"'([^']+)'") +_REFERENCE_CALL_PATTERN: re.Pattern[str] = re.compile( + r"__(?:ref|source|seed)\(\s*['\"]([^'\"]+)['\"]\s*\)" +) +_EMPTY_FIXTURE_CTE_PATTERN: re.Pattern[str] = re.compile( + r"(?P\b__(?:ref|source|seed|expected)__(?P\w+)\s+AS\s*\(\s*SELECT\s+\*\s+FROM\s+)" + r"__empty_fixture\s*\(\s*\)", + flags=re.IGNORECASE, +) +_EMPTY_FIXTURE_CALL_PATTERN: re.Pattern[str] = re.compile( + r"\b__empty_fixture\s*\(\s*\)", flags=re.IGNORECASE +) +_OPEN_EMPTY_FIXTURE_RELATION: str = "__sqlbuild_empty_fixture" +_BUILT_IN_AUDIT_ROOT: str = "" +_COLUMN_ONLY_BUILT_IN_AUDITS: frozenset[str] = frozenset({"not_null", "unique"}) +_UNKNOWN_TABLE_MESSAGE_PREFIX: str = "Unknown table " +_SYNTAX_ERROR_CODE: str = "P001" +_RESOURCE_KIND_LABELS: dict[CompiledResourceType, str] = { + CompiledResourceType.AUDIT: "audit", + CompiledResourceType.SQL_TEST: "SQL test", + CompiledResourceType.MODEL: "model", +} + + +@dataclass(frozen=True) +class _ResourceSql: + """One authored SQL body owned by a non-model resource.""" + + resource_type: CompiledResourceType + resource_name: str + path: Path + contents: str + authored_body: str + sql: str + relations: frozenset[str] + label: str | None = None + audit_definition: str | None = None + opt_out: SourceLocation | None = None + + +@dataclass(frozen=True) +class _HelpContext: + shapes: dict[str, dict[str, str]] + dialect: str | None + + +def get_resource_sql_diagnostics( + *, + project: CompiledProject, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> tuple[CompilerDiagnostic, ...]: + """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" + + resources: tuple[_ResourceSql, ...] = tuple( + resource + for resource in dict.fromkeys( + ( + *_audit_resources(project=project, shapes=shapes, profile=profile), + *_sql_test_resources(project=project), + *_hook_resources(project=project), + ) + ) + if resource.opt_out is None or project.settings.require_sql_analysis + ) + if not resources: + return () + dialect: str | None = profile.sql_analysis_dialect + normalized: dict[str, str] = {} + for resource in resources: + if resource.sql not in normalized: + normalized[resource.sql] = normalize_analysis_sql( + sql=cursor_intrinsics_analysis_sql(sql=resource.sql, cursor_type=None), + dialect=dialect, + ) + cleaned: tuple[str, ...] = tuple(normalized[resource.sql] for resource in resources) + known_functions: tuple[str, ...] = known_function_names(project.functions) + known_types: tuple[str, ...] = known_declared_types( + functions=project.functions, column_types=shapes + ) + requests: dict[tuple[str, frozenset[str]], SqlSchemaValidationRequest] = {} + for resource, cleaned_sql in zip(resources, cleaned, strict=True): + _ = requests.setdefault( + (cleaned_sql, resource.relations), + SqlSchemaValidationRequest( + sql=cleaned_sql, + dialect=dialect, + schema=_relation_schema(relations=resource.relations, shapes=shapes), + known_functions=known_functions, + known_types=known_types, + quoted_identifiers_ignore_case=profile.quoted_identifiers_ignore_case, + catalog=project.binding_catalog, + ), + ) + unique_results: dict[tuple[str, frozenset[str]], SqlBindingResult] = dict( + zip(requests, get_schema_validations(requests=tuple(requests.values())), strict=True) + ) + results: tuple[SqlBindingResult, ...] = tuple( + unique_results[(cleaned_sql, resource.relations)] + for resource, cleaned_sql in zip(resources, cleaned, strict=True) + ) + diagnostics: dict[tuple[object, ...], CompilerDiagnostic] = {} + for resource, cleaned_sql, result in zip(resources, cleaned, results, strict=True): + found: dict[tuple[object, ...], CompilerDiagnostic] = _resource_diagnostics( + resource=resource, + cleaned_sql=cleaned_sql, + result=result, + context=_HelpContext(shapes=shapes, dialect=dialect), + ) + if resource.opt_out is None: + for key, diagnostic in found.items(): + diagnostics.setdefault(key, diagnostic) + elif not any(item.code == _SYNTAX_ERROR_CODE for item in found.values()): + diagnostics.setdefault( + (resource.opt_out.path, resource.opt_out.line, resource.opt_out.column), + unneeded_opt_out_diagnostic( + resource_type=resource.resource_type, + kind=_RESOURCE_KIND_LABELS[resource.resource_type], + name=resource.resource_name, + location=resource.opt_out, + hidden=tuple(found.values()), + ), + ) + return tuple(diagnostics.values()) + + +def _resource_diagnostics( + *, + resource: _ResourceSql, + cleaned_sql: str, + result: SqlBindingResult, + context: _HelpContext, +) -> dict[tuple[object, ...], CompilerDiagnostic]: + found: dict[tuple[object, ...], CompilerDiagnostic] = {} + for index, native in enumerate(result.diagnostics): + diagnostic: SqlBindingDiagnostic | None = _reported_diagnostic( + resource=resource, diagnostic=native + ) + if diagnostic is None: + continue + location: SourceLocation = _location( + resource=resource, cleaned_sql=cleaned_sql, diagnostic=diagnostic + ) + key: tuple[object, ...] = ( + resource.path, + diagnostic.code, + diagnostic.message, + location.line, + location.column, + index if diagnostic.start is None else -1, + ) + found.setdefault( + key, + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity(diagnostic.severity), + code=diagnostic.code, + message=diagnostic.message + if resource.label is None or diagnostic.code == _SYNTAX_ERROR_CODE + else f"{resource.label}: {diagnostic.message}", + resource_type=resource.resource_type, + resource_name=resource.resource_name, + location=location, + help=resource_sql_help( + resource_type=resource.resource_type, + diagnostic=diagnostic, + authored_text=_text_from(resource=resource, location=location), + authored_line=_line_at(resource=resource, location=location), + column=location.column, + authored_body=resource.authored_body, + audit_definition=resource.audit_definition, + shapes=context.shapes, + relations=resource.relations, + dialect=context.dialect, + ), + ), + ) + return found + + +def _relation_schema( + *, relations: frozenset[str], shapes: dict[str, dict[str, str]] +) -> dict[str, dict[str, str]]: + """Return closed shapes for referenced relations; an empty shape stays open.""" + + return {name: shapes.get(name) or {} for name in relations} + + +def _reported_diagnostic( + *, resource: _ResourceSql, diagnostic: SqlBindingDiagnostic +) -> SqlBindingDiagnostic | None: + """Drop open-relation lookups and turn parser rejections into syntax errors.""" + + if diagnostic.code != BINDING_UNKNOWN_TABLE_INTERNAL_CODE: + return diagnostic + if diagnostic.message.startswith(_UNKNOWN_TABLE_MESSAGE_PREFIX): + return None + return SqlBindingDiagnostic( + code=_SYNTAX_ERROR_CODE, + message=( + f"SQL syntax error in {_RESOURCE_KIND_LABELS[resource.resource_type]} " + f"'{resource.resource_name}': {diagnostic.message}" + ), + line=diagnostic.line, + column=diagnostic.column, + start=diagnostic.start, + end=diagnostic.end, + ) + + +def _audit_resources( + *, + project: CompiledProject, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> tuple[_ResourceSql, ...]: + analysis_disabled: frozenset[str] = _analysis_disabled_models(project) + resources: list[_ResourceSql] = [] + for audit in project.audits: + if audit.attached_target_name in analysis_disabled or _binds_cleanly( + audit=audit, shapes=shapes, profile=profile + ): + continue + relations: frozenset[str] = _relations(audit.references) + authored_bodies: tuple[str | None, ...] = ( + audit.audit_block.sql_body, + audit.audit_block.measure_sql, + audit.audit_block.evidence_sql, + ) + for sql, authored in zip( + (audit.sql_body, audit.measure_sql, audit.evidence_sql), authored_bodies, strict=True + ): + if not sql or not sql.strip(): + continue + body: str = authored or sql + resources.append( + _ResourceSql( + resource_type=CompiledResourceType.AUDIT, + resource_name=audit.name, + path=audit.audit_file.relative_path, + contents=audit.audit_file.contents, + authored_body=body, + sql=sql, + relations=relations, + label=_audit_label(audit), + audit_definition=audit.definition_name, + opt_out=_header_opt_out( + header_values=audit.audit_block.header_values, + path=audit.audit_file.relative_path, + contents=audit.audit_file.contents, + body=body, + ), + ) + ) + return tuple(resources) + + +def _binds_cleanly( + *, audit: CompiledAudit, shapes: dict[str, dict[str, str]], profile: ExpressionInferenceProfile +) -> bool: + """A built-in single-column audit on a column its closed target shape contains.""" + + if ( + audit.definition_name not in _COLUMN_ONLY_BUILT_IN_AUDITS + or audit.audit_file.file_path.parts[:1] != (_BUILT_IN_AUDIT_ROOT,) + or audit.attached_target_name is None + or audit.attached_column_name is None + ): + return False + shape: dict[str, str] = shapes.get(audit.attached_target_name) or {} + return bool(shape) and ( + resolve_binding_column( + name=audit.attached_column_name, + columns=shape, + dialect=profile.sql_analysis_dialect, + ignore_quoted_case=profile.quoted_identifiers_ignore_case, + ) + is not None + ) + + +def _audit_label(audit: CompiledAudit) -> str: + label: str = f"audit '{audit.definition_name}'" + if audit.attached_target_name is not None: + label += f" on {audit.attached_target_kind} '{audit.attached_target_name}'" + if audit.attached_column_name is not None: + label += f" column '{audit.attached_column_name}'" + return label + + +def _sql_test_resources(*, project: CompiledProject) -> tuple[_ResourceSql, ...]: + analysis_disabled: frozenset[str] = _analysis_disabled_models(project) + resources: list[_ResourceSql] = [] + prepared: dict[str, tuple[str, frozenset[str]]] = {} + for test in project.sql_tests: + if not test.sql_body.strip() or analysis_disabled.intersection(test.expected_model_names): + continue + if test.sql_body not in prepared: + prepared[test.sql_body] = _prepared_test_sql(test.sql_body) + sql, relations = prepared[test.sql_body] + resources.append( + _ResourceSql( + resource_type=CompiledResourceType.SQL_TEST, + resource_name=test.name, + path=test.test_file.relative_path, + contents=test.test_file.contents, + authored_body=test.test_block.sql_body, + sql=sql, + label=f"SQL test '{test.name}'", + opt_out=_header_opt_out( + header_values=test.test_block.header_values, + path=test.test_file.relative_path, + contents=test.test_file.contents, + body=test.test_block.sql_body, + ), + relations=relations, + ) + ) + return tuple(resources) + + +def _prepared_test_sql(sql_body: str) -> tuple[str, frozenset[str]]: + """Test SQL with empty fixtures opened, and the relations it reads.""" + + sql: str = _EMPTY_FIXTURE_CTE_PATTERN.sub( + lambda match: match.group("prefix") + match.group("name"), sql_body + ) + return _EMPTY_FIXTURE_CALL_PATTERN.sub(_OPEN_EMPTY_FIXTURE_RELATION, sql), frozenset( + ( + *_REFERENCE_CALL_PATTERN.findall(sql_body), + *(match.group("name") for match in _EMPTY_FIXTURE_CTE_PATTERN.finditer(sql_body)), + ) + ) + + +def _hook_resources(*, project: CompiledProject) -> tuple[_ResourceSql, ...]: + hook_files: dict[Path, str] = { + hook_file.relative_path: hook_file.contents for hook_file in project.sql_hook_files + } + resources: list[_ResourceSql] = [] + for model in project.models: + if model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False: + continue + for hook_key in _MODEL_HOOK_KEYS: + entries: object = model.config.values.get(hook_key) + if not isinstance(entries, list | tuple): + continue + for hook_index, entry in enumerate(entries): + statement: object = getattr(entry, "statement", None) + if not isinstance(statement, str) or not statement.strip(): + continue + hook_path: Path | None = getattr(entry, "relative_path", None) + definition: object = getattr(entry, "definition_sql", None) + named: bool = hook_path is not None and hook_path in hook_files + resources.append( + _ResourceSql( + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + path=hook_path if named and hook_path is not None else model.relative_path, + contents=hook_files[hook_path] + if named and hook_path is not None + else model.authored_sql, + authored_body=definition + if named and isinstance(definition, str) + else statement, + sql=statement, + relations=_relations(tuple(getattr(entry, "reads", ()) or ())) + | frozenset(_REFERENCE_CALL_PATTERN.findall(statement)), + label=f"{hook_key}[{hook_index}] SQL hook", + ) + ) + return tuple(resources) + + +def _analysis_disabled_models(project: CompiledProject) -> frozenset[str]: + return frozenset( + model.name + for model in project.models + if model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False + ) + + +def _header_opt_out( + *, header_values: dict[str, object], path: Path, contents: str, body: str +) -> SourceLocation | None: + """Where a TEST or AUDIT header turns SQL analysis off, if it does.""" + + if header_values.get(SQL_ANALYSIS_CONFIG_KEY) is not False: + return None + body_offset: int = contents.find(body) + key_offset: int = contents.rfind( + SQL_ANALYSIS_CONFIG_KEY, 0, body_offset if body_offset >= 0 else len(contents) + ) + return _offset_location(path=path, text=contents, offset=max(key_offset, 0)) + + +def _relations(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: + return binding_relation_names( + tuple(reference for reference in references if isinstance(reference, CompileSqlReference)) + ) + + +def _location( + *, resource: _ResourceSql, cleaned_sql: str, diagnostic: SqlBindingDiagnostic +) -> SourceLocation: + """Point at the offending authored text, falling back to the body start.""" + + body_offset: int = max(resource.contents.find(resource.authored_body), 0) + if diagnostic.start is None: + name_offset: int | None = _quoted_name_offset( + text=resource.contents, start=body_offset, message=diagnostic.message + ) + if name_offset is not None: + return _offset_location(path=resource.path, text=resource.contents, offset=name_offset) + location: SourceLocation | None = get_authored_binding_location( + path=resource.path, + authored_sql=resource.contents, + authored_query_sql=resource.authored_body, + cleaned_sql=cleaned_sql, + diagnostic=diagnostic, + expansion=CompiledSqlExpansion( + authored_sql=resource.authored_body, expanded_sql=resource.sql, passes=() + ), + ) + if location is not None: + return location + return _offset_location(path=resource.path, text=resource.contents, offset=body_offset) + + +def _quoted_name_offset(*, text: str, start: int, message: str) -> int | None: + for name in _QUOTED_NAME_PATTERN.findall(message): + identifier: str = name.rsplit(".", 1)[-1] + match: re.Match[str] | None = re.search( + rf"(? str: + """Authored text of a diagnostic span (or from its start), for help that quotes the source.""" + + if location.path != resource.path: + return "" + start: int | None = _offset(text=resource.contents, line=location.line, column=location.column) + if start is None: + return "" + end: int | None = ( + _offset(text=resource.contents, line=location.end_line, column=location.end_column) + if location.end_line is not None and location.end_column is not None + else None + ) + return resource.contents[start : end if end is not None and end > start else None] + + +def _line_at(*, resource: _ResourceSql, location: SourceLocation) -> str: + lines: list[str] = resource.contents.splitlines() + if location.path != resource.path or not 1 <= location.line <= len(lines): + return "" + return lines[location.line - 1] + + +def _offset(*, text: str, line: int, column: int) -> int | None: + lines: list[str] = text.splitlines(keepends=True) + if not 1 <= line <= len(lines): + return None + return sum(len(value) for value in lines[: line - 1]) + max(column - 1, 0) + + +def _offset_location(*, path: Path, text: str, offset: int) -> SourceLocation: + line: int = text.count("\n", 0, offset) + 1 + column: int = offset - (text.rfind("\n", 0, offset) + 1) + 1 + return SourceLocation(path=path, line=line, column=column) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py new file mode 100644 index 0000000000..e9f3c5f3ff --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py @@ -0,0 +1,232 @@ +"""Concrete remediation for compile checks on SQL tests and audits.""" + +from __future__ import annotations + +import re + +from sqlbuild.compiler.compile._helpers.diagnostics.details import closest_column, missing_column +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + +_SET_OPERATION_CODE: str = "B215" +_COMPARISON_CODE: str = "B217" +_UNKNOWN_COLUMN_CODE: str = "B002" +_SNOWFLAKE: str = "snowflake" +_SET_OPERATION_TYPES: re.Pattern[str] = re.compile( + r"column (?P\d+).*accumulated type (?P[\w()]+), next type (?P[\w()]+)", + re.IGNORECASE, +) +_COMPARISON_TYPES: re.Pattern[str] = re.compile( + r"between (?P[\w()]+) and (?P[\w()]+)", re.IGNORECASE +) +_ALIAS_NAME: re.Pattern[str] = re.compile( + r"\s+AS\s+\"?(?P[A-Za-z_][\w$]*)\"?,?$", re.IGNORECASE +) +_ALIAS_SUFFIX: re.Pattern[str] = re.compile(r"\s+AS\s+\"?[A-Za-z_][\w$]*\"?,?$", re.IGNORECASE) +_CAST: re.Pattern[str] = re.compile(r"CAST\((?P.+)\s+AS\s+[\w(), ]+\)", re.IGNORECASE) +_RELATION_PARAMETER: str = "@relation" +_LISTED_COLUMNS: int = 10 +_SINGLE_OPERAND: re.Pattern[str] = re.compile( + r"'(?:[^']|'')*'|[A-Za-z_][\w$]*(?:\.[A-Za-z_][\w$]*)?" +) +_WORD_CHARACTER: re.Pattern[str] = re.compile(r"[\w$]") +_PARAMETER: re.Pattern[str] = re.compile(r"@[A-Za-z_]\w*") +_TEXT_TYPES: tuple[str, ...] = ("VARCHAR", "STRING", "TEXT", "CHAR") +_TEMPORAL_TYPES: tuple[str, ...] = ("TIMESTAMP", "DATE") +_SET_OPERATION_KEYWORDS: frozenset[str] = frozenset( + {"UNION", "ALL", "EXCEPT", "INTERSECT", "MINUS", "SELECT", "FROM", "DISTINCT", "BY", "NAME"} +) + + +def resource_sql_help( + *, + resource_type: CompiledResourceType, + diagnostic: SqlBindingDiagnostic, + authored_text: str, + authored_line: str, + column: int, + authored_body: str, + audit_definition: str | None, + shapes: dict[str, dict[str, str]], + relations: frozenset[str], + dialect: str | None, +) -> str | None: + """Return a concrete fix quoting the authored span, or None to keep the generic help.""" + + if diagnostic.code == _SET_OPERATION_CODE: + return _set_operation_help( + message=diagnostic.message, + authored_text=authored_text, + authored_line=authored_line, + span_column=column, + sql_test=resource_type == CompiledResourceType.SQL_TEST, + relation_shapes={name: shapes.get(name) or {} for name in relations}, + ) + if diagnostic.code == _COMPARISON_CODE and resource_type == CompiledResourceType.AUDIT: + return _audit_comparison_help( + message=diagnostic.message, + authored_line=authored_line, + column=column, + authored_body=authored_body, + audit_definition=audit_definition, + dialect=dialect, + ) + if diagnostic.code == _UNKNOWN_COLUMN_CODE and resource_type == CompiledResourceType.SQL_TEST: + return _unknown_column_help(message=diagnostic.message, shapes=shapes) + return None + + +def _set_operation_help( + *, + message: str, + authored_text: str, + authored_line: str, + span_column: int, + sql_test: bool, + relation_shapes: dict[str, dict[str, str]], +) -> str | None: + match: re.Match[str] | None = _SET_OPERATION_TYPES.search(message) + if match is None: + return None + position: str = match.group("position") + left: str = match.group("left").upper() + right: str = match.group("right").upper() + column: str = _set_operation_operand( + authored_text=authored_text, + authored_line=authored_line, + column=span_column, + position=position, + ) + if not sql_test: + return ( + "Cast one branch so both branches have the same type, for example " + f"`CAST({column} AS {left})` for set-operation column {position}" + ) + model_type: str | None = _model_type( + authored_text=authored_text, operand=column, relation_shapes=relation_shapes + ) + if model_type is not None: + return ( + f"Cast set-operation column {position} to the tested model's type {model_type} in " + f"the expected or fixture rows, for example `CAST({column} AS {model_type})`" + ) + target: str = right if _is_text(left) and not _is_text(right) else left + return ( + f"Cast set-operation column {position} explicitly so every branch has the same type, " + f"for example `CAST({column} AS {target})` in the expected or fixture rows" + ) + + +def _model_type( + *, authored_text: str, operand: str, relation_shapes: dict[str, dict[str, str]] +) -> str | None: + """The tested relation's type for the branch column, when exactly one type matches.""" + + names: list[str] = [] + alias: re.Match[str] | None = _ALIAS_NAME.search(authored_text.strip()) + if alias is not None: + names.append(alias.group("name")) + names.append(operand.rsplit(".", 1)[-1].strip('"')) + for name in names: + found: set[str] = set() + for shape in relation_shapes.values(): + for output, value in shape.items(): + if output.casefold() == name.casefold() and value: + found.add(value.split("(", 1)[0].upper()) + if len(found) == 1: + return found.pop() + return None + + +def _audit_comparison_help( + *, + message: str, + authored_line: str, + column: int, + authored_body: str, + audit_definition: str | None, + dialect: str | None, +) -> str | None: + match: re.Match[str] | None = _COMPARISON_TYPES.search(message) + if match is None: + return None + types: tuple[str, str] = (match.group("left").upper(), match.group("right").upper()) + temporal: str | None = _temporal_type(types) + if temporal is None or not any(_is_text(value) for value in types): + return None + operand: str = _audit_parameter( + authored_line=authored_line, column=column, authored_body=authored_body + ) + conversion: str = ( + f"TO_{temporal}({operand})" if dialect == _SNOWFLAKE else f"CAST({operand} AS {temporal})" + ) + audit: str = f"audit '{audit_definition}'" if audit_definition else "the audit" + return ( + f"Convert the text value explicitly, for example `{conversion}`, " + f"or attach {audit} to a {temporal} column" + ) + + +def _set_operation_operand( + *, authored_text: str, authored_line: str, column: int, position: str +) -> str: + """The authored branch operand when the span is exactly one, else the VALUES column name.""" + + preceding: str = authored_line[column - 2 : column - 1] if column > 1 else "" + expression: str = _ALIAS_SUFFIX.sub("", authored_text.strip().rstrip(",")) + cast: re.Match[str] | None = _CAST.fullmatch(expression) + if cast is not None: + expression = cast.group("inner").strip() + if ( + not _WORD_CHARACTER.fullmatch(preceding) + and _SINGLE_OPERAND.fullmatch(expression) + and expression.upper() not in _SET_OPERATION_KEYWORDS + ): + return expression + return f"COLUMN{position}" + + +def _audit_parameter(*, authored_line: str, column: int, authored_body: str) -> str: + """The audit parameter nearest the finding on its line, else the body's only parameter.""" + + on_line: list[re.Match[str]] = [ + match + for match in _PARAMETER.finditer(authored_line) + if match.group() != _RELATION_PARAMETER + ] + if on_line: + return min(on_line, key=lambda match: abs(match.start() + 1 - column)).group() + parameters: set[str] = set(_PARAMETER.findall(authored_body)) - {_RELATION_PARAMETER} + return next(iter(parameters)) if len(parameters) == 1 else "" + + +def _unknown_column_help(*, message: str, shapes: dict[str, dict[str, str]]) -> str | None: + missing: tuple[str, str | None] | None = missing_column(message) + if missing is None: + return None + name: str = missing[0] + table: str | None = missing[1] + if table is None: + return None + columns: dict[str, str] = shapes.get(table, {}) + suggestion: str | None = closest_column(name=name, columns=columns) + if suggestion is not None: + return f"did you mean '{suggestion}'?" + listed: str = ", ".join(list(columns)[:_LISTED_COLUMNS]) + more: str = ( + f", and {len(columns) - _LISTED_COLUMNS} more" if len(columns) > _LISTED_COLUMNS else "" + ) + outputs: str = f" (it outputs {listed}{more})" if columns else "" + return f"'{table}' does not output '{name}'{outputs}; update the test to its current columns" + + +def _temporal_type(types: tuple[str, str]) -> str | None: + for value in types: + for kind in _TEMPORAL_TYPES: + if value.startswith(kind): + return kind + return None + + +def _is_text(value: str) -> bool: + return value.startswith(_TEXT_TYPES) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py new file mode 100644 index 0000000000..020aa3568e --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py @@ -0,0 +1,222 @@ +"""Reject `sql_analysis false` on parseable models when the project requires SQL analysis.""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import replace +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.analysis.validation import ( + validate_hook_sql_syntax, + validate_sql_syntax, +) +from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompilerDiagnostic, + SqlAnalysisOptOutRequest, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.constants import ( + PROJECT_CONFIG_FILENAME, + REQUIRE_SQL_ANALYSIS_SETTING_KEY, + SETTINGS_SECTION, + SQL_ANALYSIS_CONFIG_KEY, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.setting_help import setting_help +from sqlbuild.errors.setting_help.main.setting_note import setting_note +from sqlbuild.presentation.main.count_noun import format_count_noun +from sqlbuild.spec.contracts.models import SettingsConfig, SourceLocation + +_LEGACY_KEY: str = "sql_validation" +_SEMANTIC_CODE_PREFIX: str = "B" +_OPT_OUT_KEYS: tuple[str, ...] = (SQL_ANALYSIS_CONFIG_KEY, _LEGACY_KEY) +_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +_OPT_OUT_KEY_PATTERN: re.Pattern[str] = re.compile(r"\b(?:sql_analysis|sql_validation)\b") +_FINDING_KINDS: tuple[tuple[re.Pattern[str], str, str], ...] = ( + (re.compile(r"^B002$"), "unknown column", "unknown columns"), + (re.compile(r"^B003$"), "ambiguous column", "ambiguous columns"), + (re.compile(r"^B00[45]$"), "reference error", "reference errors"), + (re.compile(r"^B101$"), "unknown function", "unknown functions"), + (re.compile(r"^B102$"), "function arity error", "function arity errors"), + (re.compile(r"^B216$"), "column count mismatch", "column count mismatches"), + (re.compile(r"^B21\d$"), "type mismatch", "type mismatches"), + (re.compile(r"^B23[0-2]$"), "grouping or window error", "grouping or window errors"), + (re.compile(r"^B3\d\d$"), "metadata mismatch", "metadata mismatches"), +) + + +def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLocation | None: + """Return where a model turns SQL analysis off when the project forbids it for that model.""" + + settings: SettingsConfig = request.settings + if ( + not settings.require_sql_analysis + or not settings.sql_analysis + or request.no_sql_validation + or not any(request.config.values.get(key) is False for key in _OPT_OUT_KEYS) + ): + return None + model_name: str = request.model_file.file_path.stem + try: + validate_sql_syntax( + query_sql=request.query_sql, + model_name=model_name, + file_path=request.model_file.file_path, + placeholders=request.placeholders, + ) + for hook_name in _HOOK_KEYS: + validate_hook_sql_syntax( + value=request.config.values.get(hook_name), + hook_name=hook_name, + model_name=model_name, + file_path=request.model_file.file_path, + placeholders=request.placeholders, + ) + except CompileInputError: + return None + return _opt_out_location(request) + + +def reject_unneeded_sql_analysis_opt_outs(project: CompiledProject) -> CompiledProject: + """Replace each rejected opt-out model's findings with one error that counts them.""" + + rejected: dict[str, CompiledModel] = { + model.name: model + for model in project.models + if model.rejected_sql_analysis_opt_out is not None + } + if not rejected: + return project + hidden: dict[str, list[CompilerDiagnostic]] = {name: [] for name in rejected} + kept: list[CompilerDiagnostic] = [] + for diagnostic in project.diagnostics: + if ( + diagnostic.resource_name in hidden + and diagnostic.resource_type in (None, CompiledResourceType.MODEL) + and diagnostic.code.startswith(_SEMANTIC_CODE_PREFIX) + ): + hidden[diagnostic.resource_name].append(diagnostic) + else: + kept.append(diagnostic) + kept.extend( + _opt_out_diagnostic(model=model, hidden=tuple(hidden[name])) + for name, model in sorted(rejected.items()) + ) + return replace(project, diagnostics=tuple(kept)) + + +def _opt_out_diagnostic( + *, model: CompiledModel, hidden: tuple[CompilerDiagnostic, ...] +) -> CompilerDiagnostic: + location: SourceLocation | None = model.rejected_sql_analysis_opt_out + return unneeded_opt_out_diagnostic( + resource_type=CompiledResourceType.MODEL, + kind="model", + name=model.name, + location=location, + hidden=hidden, + path_default=location is not None and location.path.name == PROJECT_CONFIG_FILENAME, + ) + + +def unneeded_opt_out_diagnostic( + *, + resource_type: CompiledResourceType, + kind: str, + name: str, + location: SourceLocation | None, + hidden: tuple[CompilerDiagnostic, ...], + path_default: bool = False, +) -> CompilerDiagnostic: + """One P009 error for `sql_analysis false` on SQL that parses, counting what it hid.""" + + findings: str = _finding_counts(hidden) + opt_out: str = ( + "`sql_analysis = false` from this [path_defaults] entry" + if path_default + else "`sql_analysis false`" + ) + fix: str = ( + f"remove {opt_out} and fix the findings it was hiding: {findings} " + "(run `sqb compile` to see them)" + if findings + else f"remove {opt_out}; it is not hiding any findings" + ) + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE, + message=f"`sql_analysis false` is not needed for {kind} '{name}'", + resource_type=resource_type, + resource_name=name, + location=location, + notes=( + setting_note( + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, + value=True, + ) + + ", which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; " + f"this {kind} parses successfully.", + ), + help=join_helps( + fix, + setting_help( + purpose="to allow `sql_analysis false` on any model, SQL test or audit", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, + value=False, + ), + ), + ) + + +def _finding_counts(hidden: tuple[CompilerDiagnostic, ...]) -> str: + counts: Counter[tuple[str, str]] = Counter(_finding_kind(item.code) for item in hidden) + return ", ".join( + format_count_noun(count=count, singular=singular, plural=plural) + for (singular, plural), count in sorted(counts.items(), key=lambda item: -item[1]) + ) + + +def _finding_kind(code: str) -> tuple[str, str]: + for pattern, singular, plural in _FINDING_KINDS: + if pattern.match(code): + return singular, plural + return "other finding", "other findings" + + +def _opt_out_location(request: SqlAnalysisOptOutRequest) -> SourceLocation: + model_file: DiscoveredSqlModelFile = request.model_file + if any(key in request.config.model_header_keys for key in _OPT_OUT_KEYS): + return _text_location(path=model_file.relative_path, text=model_file.contents, start=0) + path_key: str | None = request.config.matched_path_default + config_text: str = ( + request.project_config_path.read_text(encoding="utf-8") + if request.project_config_path.is_file() + else "" + ) + start: int = config_text.find(path_key) if path_key else -1 + return _text_location( + path=Path(request.project_config_path.name), text=config_text, start=max(start, 0) + ) + + +def _text_location(*, path: Path, text: str, start: int) -> SourceLocation: + match: re.Match[str] | None = _OPT_OUT_KEY_PATTERN.search(text, start) + offset: int = match.start() if match is not None else 0 + line: int = text.count("\n", 0, offset) + 1 + column: int = offset - (text.rfind("\n", 0, offset) + 1) + 1 + return SourceLocation(path=path, line=line, column=column) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index 9ce894e4a0..dfa069eb6c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -18,6 +18,7 @@ DiagnosticPhase, DiagnosticSeverity, ) +from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind @@ -156,8 +157,7 @@ def reject_macro_generated_references( help=( f"write the reference in {location}, or pass it in: " f"@{loaded_macro.name}({reference_call_text(generated)})\n" - " = help: while migrating a project, allow macro-generated references with " - "[references] enforce_explicit = false in sqlbuild_project.toml" + " = help: " + explicit_references_help(allowed="macro-generated references") ), ), ) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py index 1c32f74bbe..bcafdfbbaa 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/python_sql.py @@ -16,6 +16,7 @@ PythonSqlReferenceReport, ) from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity +from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main._model_python_hook_names import model_python_hook_names from sqlbuild.compiler.references.main._compiled_project_relations import ( @@ -345,7 +346,6 @@ def _hard_coded_relation_diagnostic( location=SourceLocation(path=owner.relative_path, line=literal.line, column=1), help=( f"{remedy}{qualify_help}\n" - " = help: while migrating a project, allow hard-coded relation names with " - "[references] enforce_explicit = false in sqlbuild_project.toml" + " = help: " + explicit_references_help(allowed="hard-coded relation names") ), ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index e1d88f29fe..73d04a6510 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -14,6 +14,8 @@ MACRO_GENERATED_REFERENCE_CODE: str = "P006" HOOK_READS_OWN_DEPENDANT_CODE: str = "P007" HARD_CODED_PROJECT_RELATION_CODE: str = "P008" +UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" +SQL_ANALYSIS_OPT_OUT_ENTRY: str = "sql_analysis false" HOOK_DIRECTORY_NAME: str = "hooks" MODEL_DIRECTORY_NAME: str = "models" NOT_NULL_AUDIT_NAME: str = "not_null" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 35bbaca3d6..a40fe5f8c1 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -227,6 +227,7 @@ def build_compile_inputs( ) ), effective_settings=effective_settings, + no_sql_validation=no_sql_validation, effective_vars=effective_vars, macro_context=macro_context, loaded_macros=declaration_scope.loaded_macros, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index b8cf103dd3..dcd3fbfef1 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -734,6 +734,7 @@ class CompileModelInput: schema_entry: SchemaModelEntry | None = None schema_file: DiscoveredSchemaFile | None = None sql_validation_enabled: bool = False + rejected_sql_analysis_opt_out: SourceLocation | None = None enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) enum_columns: dict[str, EnumDeclaration] = field(default_factory=dict) @@ -851,6 +852,7 @@ class CompileProjectInputs: compile_cache_dir: Path | None = None effective_connection: dict[str, object] = field(default_factory=dict) effective_settings: SettingsConfig = field(default_factory=SettingsConfig) + no_sql_validation: bool = False effective_vars: dict[str, object] = field(default_factory=dict) macro_context: MacroContext | None = field(default=None, repr=False, compare=False) loaded_macros: dict[str, LoadedMacro] = field(default_factory=dict) @@ -910,6 +912,7 @@ class CompiledModel: binding_validated: bool = False dynamic_column_contract: DynamicColumnContractProof | None = None unchecked_output_columns: frozenset[str] = frozenset() + rejected_sql_analysis_opt_out: SourceLocation | None = None @dataclass(frozen=True) @@ -1540,3 +1543,16 @@ def get(self, raw_columns: object) -> CachedModelHeaderColumns | None: def put(self, cached: CachedModelHeaderColumns) -> None: self._values[id(cached.raw_columns)] = cached + + +@dataclass(frozen=True) +class SqlAnalysisOptOutRequest: + """One model's opt-out, with what is needed to prove its SQL parses.""" + + model_file: DiscoveredSqlModelFile + config: CompileModelConfig + settings: SettingsConfig + no_sql_validation: bool + query_sql: str + placeholders: dict[str, str] | None + project_config_path: Path diff --git a/src/sqlbuild/compiler/discovery/_helpers/settings/__init__.py b/src/sqlbuild/compiler/discovery/_helpers/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py b/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py new file mode 100644 index 0000000000..8a7f24facf --- /dev/null +++ b/src/sqlbuild/compiler/discovery/_helpers/settings/guidance.py @@ -0,0 +1,91 @@ +"""Current values of project settings and the exact TOML that changes them.""" + +from __future__ import annotations + +from sqlbuild.compiler.discovery.constants import ( + ENFORCE_EXPLICIT_REFERENCES_KEY, + LOCAL_CONFIG_FILENAME, + MICROBATCH_CONCURRENCY_SETTING_KEY, + PROJECT_CONFIG_FILENAME, + REFERENCES_SECTION, + SETTINGS_SECTION, + SQL_ANALYSIS_CONFIG_KEY, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.errors.setting_help.main.setting_help import setting_help +from sqlbuild.errors.setting_help.main.setting_note import setting_note + + +def effective_sql_analysis(discovered_inputs: DiscoveredProjectInputs) -> bool: + """Return the project SQL analysis setting after local overrides.""" + + if SQL_ANALYSIS_CONFIG_KEY in discovered_inputs.local_config.setting_overrides: + return discovered_inputs.local_config.settings.sql_analysis + return discovered_inputs.project_config.settings.sql_analysis + + +def sql_analysis_off_note_and_help( + *, discovered_inputs: DiscoveredProjectInputs, purpose: str +) -> tuple[str, str]: + """Return a note naming the file that turns SQL analysis off and help to turn it on.""" + + file_name: str = ( + LOCAL_CONFIG_FILENAME + if SQL_ANALYSIS_CONFIG_KEY in discovered_inputs.local_config.setting_overrides + else PROJECT_CONFIG_FILENAME + ) + return ( + setting_note( + file_name=file_name, section=SETTINGS_SECTION, key=SQL_ANALYSIS_CONFIG_KEY, value=False + ), + setting_help( + purpose=purpose, + file_name=file_name, + section=SETTINGS_SECTION, + key=SQL_ANALYSIS_CONFIG_KEY, + value=True, + ), + ) + + +def microbatch_concurrency_note() -> str: + """State that concurrent microbatches are off in the project file.""" + + return setting_note( + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=MICROBATCH_CONCURRENCY_SETTING_KEY, + value=False, + explicit=None, + ) + + +def microbatch_concurrency_help() -> str: + """Show the exact setting that allows `batch_concurrency` above one.""" + + return setting_help( + purpose="to run microbatches concurrently", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=MICROBATCH_CONCURRENCY_SETTING_KEY, + value=True, + ) + + +def explicit_references_help(*, allowed: str) -> str: + """Show the exact migration setting that relaxes explicit references, with its value.""" + + note: str = setting_note( + file_name=PROJECT_CONFIG_FILENAME, + section=REFERENCES_SECTION, + key=ENFORCE_EXPLICIT_REFERENCES_KEY, + value=True, + explicit=None, + ) + return f"{note}; " + setting_help( + purpose=f"while migrating a project, to allow {allowed}", + file_name=PROJECT_CONFIG_FILENAME, + section=REFERENCES_SECTION, + key=ENFORCE_EXPLICIT_REFERENCES_KEY, + value=False, + ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py b/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py index ef5b723390..f384da39b2 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/audits.py @@ -11,6 +11,7 @@ from sqlbuild.compiler.auditing.types import AuditEvaluationMode from sqlbuild.compiler.discovery._helpers.sql.header_keys import reject_unsupported_header_keys from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY from sqlbuild.compiler.discovery.exceptions import SqlAuditParseError from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock @@ -42,6 +43,7 @@ _AUDIT_SAMPLE_UNIT_HEADER_KEY, _AUDIT_THRESHOLDS_HEADER_KEY, _AUDIT_MINIMUM_SAMPLES_HEADER_KEY, + SQL_ANALYSIS_CONFIG_KEY, } ) @@ -165,6 +167,9 @@ def _parse_audit_header(*, header: str, header_line: int, file_path: Path) -> di always_run: object | None = parsed_header.get(_AUDIT_ALWAYS_RUN_HEADER_KEY) if _AUDIT_ALWAYS_RUN_HEADER_KEY in parsed_header and not isinstance(always_run, bool): raise SqlAuditParseError(f"AUDIT() always_run in '{file_path}' must be a boolean") + sql_analysis: object | None = parsed_header.get(SQL_ANALYSIS_CONFIG_KEY) + if SQL_ANALYSIS_CONFIG_KEY in parsed_header and not isinstance(sql_analysis, bool): + raise SqlAuditParseError(f"AUDIT() sql_analysis in '{file_path}' must be a boolean") return parsed_header diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index 32aff23f68..3cd3e72c19 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -15,7 +15,10 @@ parse_header_values, prepare_model_header_tokens, ) -from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN +from sqlbuild.compiler.discovery.constants import ( + SQL_ANALYSIS_CONFIG_KEY, + STATEMENT_HEADER_BODY_PATTERN, +) from sqlbuild.compiler.discovery.exceptions import SqlTestParseError from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, @@ -49,6 +52,7 @@ _TEST_CASES_HEADER_KEY, _TEST_CURSOR_START_HEADER_KEY, _TEST_CURSOR_END_HEADER_KEY, + SQL_ANALYSIS_CONFIG_KEY, } ) _PARAMETER_TYPES: tuple[SqlValueKind, ...] = ( @@ -219,6 +223,10 @@ def _parse_test_header(*, header: str, header_line: int, file_path: Path) -> dic _validate_test_name(name_value=parsed_header[_TEST_NAME_HEADER_KEY], file_path=file_path) if _TEST_MODE_HEADER_KEY in parsed_header: _validate_test_mode(mode_value=parsed_header[_TEST_MODE_HEADER_KEY], file_path=file_path) + if SQL_ANALYSIS_CONFIG_KEY in parsed_header and not isinstance( + parsed_header[SQL_ANALYSIS_CONFIG_KEY], bool + ): + raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") return parsed_header diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index f7446c12a9..4aad18539e 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -29,6 +29,9 @@ LOCAL_CONFIG_FILENAME, MODELS_DIRECTORY_NAME, PROJECT_CONFIG_FILENAME, + QUERY_CHANGE_TRACKING_SETTING_KEY, + REQUIRE_SQL_ANALYSIS_SETTING_KEY, + SETTINGS_SECTION, SQL_ANALYSIS_CONFIG_KEY, SQL_MODEL_HEADER_KEYS, TOML_FILE_SUFFIX, @@ -39,6 +42,9 @@ from sqlbuild.cost.constants import USD_PER_CREDIT_CONFIG_KEY from sqlbuild.cursor_algebra.constants import DURATION_DAY_UNIT from sqlbuild.cursor_algebra.models import Duration +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.setting_help import setting_help +from sqlbuild.errors.setting_help.main.setting_note import setting_note from sqlbuild.spec.contracts.constants import ( SCENARIO_RUN_NAMESPACE_CONFIG_KEY, TIME_TRAVEL_RETENTION_MATERIALIZATIONS, @@ -209,8 +215,20 @@ def load_project_config(*, project_dir: Path) -> ProjectConfig: sinks: SinksConfig = _load_sinks(payload=payload.get("sinks"), file_path=file_path) if janitor.enabled and janitor.delete_tracked_only and not settings.query_change_tracking: raise ProjectConfigError( - f"{file_path} janitor.delete_tracked_only requires " - "settings.query_change_tracking to be true" + f"{file_path} [janitor] delete_tracked_only = true requires query change tracking; " + + setting_note( + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=QUERY_CHANGE_TRACKING_SETTING_KEY, + value=False, + ), + help=setting_help( + purpose="to track which relations SQLBuild created", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=QUERY_CHANGE_TRACKING_SETTING_KEY, + value=True, + ), ) return ProjectConfig( @@ -426,6 +444,40 @@ def load_local_config(*, project_dir: Path) -> LocalConfig: ) +def validate_local_sql_analysis_policy( + *, project_dir: Path, project_config: ProjectConfig, local_config: LocalConfig +) -> None: + """Reject a local project-wide SQL analysis opt-out when the project requires analysis.""" + + if ( + not project_config.settings.require_sql_analysis + or SQL_ANALYSIS_CONFIG_KEY not in local_config.setting_overrides + or local_config.settings.sql_analysis + ): + return + raise ProjectConfigError( + f"{_resolve_local_config_path(project_dir=project_dir)} turns SQL analysis off with " + f"[settings] {SQL_ANALYSIS_CONFIG_KEY} = false, which this project does not allow; " + + setting_note( + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, + value=True, + ), + help=join_helps( + f"remove {SQL_ANALYSIS_CONFIG_KEY} from [settings] in {LOCAL_CONFIG_FILENAME}; " + "to skip SQL analysis for one run, use `--no-sql-analysis`", + setting_help( + purpose="to allow turning SQL analysis off locally", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, + value=False, + ), + ), + ) + + def _resolve_project_config_path(*, project_dir: Path) -> Path: toml_path: Path = project_dir / PROJECT_CONFIG_FILENAME if toml_path.exists(): @@ -555,6 +607,9 @@ def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: else LEGACY_SQL_VALIDATION_CONFIG_KEY ) sql_analysis: bool = _optional_bool(mapping=mapping, key=analysis_key, default=True) + require_sql_analysis: bool = _optional_bool( + mapping=mapping, key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, default=False + ) query_change_tracking: bool = _optional_bool( mapping=mapping, key="query_change_tracking", @@ -611,6 +666,7 @@ def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: ) return SettingsConfig( sql_analysis=sql_analysis, + require_sql_analysis=require_sql_analysis, query_change_tracking=query_change_tracking, column_contract_mode=column_contract_mode, concurrency=concurrency, @@ -827,6 +883,21 @@ def _load_local_settings( mapping: dict[str, object] = _coerce_mapping( payload=payload, label="settings", file_path=file_path ) + if REQUIRE_SQL_ANALYSIS_SETTING_KEY in mapping: + raise ProjectConfigError( + f"{file_path} cannot set [settings] {REQUIRE_SQL_ANALYSIS_SETTING_KEY}; " + f"it is a shared project policy that only {PROJECT_CONFIG_FILENAME} can set", + help=join_helps( + f"remove {REQUIRE_SQL_ANALYSIS_SETTING_KEY} from {LOCAL_CONFIG_FILENAME}", + setting_help( + purpose="to change the policy for everyone", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, + value=not bool(mapping[REQUIRE_SQL_ANALYSIS_SETTING_KEY]), + ), + ), + ) setting_names: frozenset[str] = frozenset(field.name for field in fields(SettingsConfig)) normalized_overrides: set[str] = set() key: str @@ -1631,7 +1702,8 @@ def _load_local_dbt(*, payload: object, file_path: Path) -> LocalDbtConfig: mapping: dict[str, object] = _coerce_mapping(payload=payload, label="dbt", file_path=file_path) if DBT_DEFER_CLONE_CONFIG_KEY in mapping: raise ProjectConfigError( - f"{file_path} [dbt].defer_clone_from was removed; use dbt-native --state/--defer" + f"{file_path} [dbt] defer_clone_from was removed; " + "remove it and use dbt-native --state/--defer" ) _validate_allowed_keys( mapping=mapping, diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 707b10863c..5777e15c13 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -122,6 +122,17 @@ SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" LEGACY_SQL_VALIDATION_CONFIG_KEY: str = "sql_validation" SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" +REQUIRE_SQL_ANALYSIS_SETTING_KEY: str = "require_sql_analysis" +SETTINGS_SECTION: str = "settings" +QUERY_CHANGE_TRACKING_SETTING_KEY: str = "query_change_tracking" +MICROBATCH_CONCURRENCY_SETTING_KEY: str = "microbatch_concurrency" +REFERENCES_SECTION: str = "references" +TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" +STAGED_PROMOTION_MODE: str = "staged" +DBT_SECTION: str = "dbt" +DEFAULTS_SECTION: str = "defaults" +DBT_PROJECT_DIR_KEY: str = "project_dir" +ENFORCE_EXPLICIT_REFERENCES_KEY: str = "enforce_explicit" DBT_LEGACY_REUSE_FROM_CONFIG_KEY: str = "reuse_from" DBT_PRODUCTION_REF_CONFIG_KEY: str = "production_ref" DBT_DEFER_CLONE_CONFIG_KEY: str = "defer_clone_from" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index 0805f693a7..7d749b96f7 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -15,6 +15,7 @@ from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, + validate_local_sql_analysis_policy, ) from sqlbuild.compiler.discovery.constants import ( DISCOVERY_FACT_CACHE_ALGORITHM, @@ -70,6 +71,9 @@ def _assemble_discovered_project_inputs( ) -> DiscoveredProjectInputs: project_config: ProjectConfig = load_project_config(project_dir=project_dir) local_config: LocalConfig = load_local_config(project_dir=project_dir) + validate_local_sql_analysis_policy( + project_dir=project_dir, project_config=project_config, local_config=local_config + ) validate_project_python_paths(project_dir=project_dir) sql_analysis_enabled: bool = ( sql_analysis_enabled_override diff --git a/src/sqlbuild/compiler/discovery/main/explicit_references_help.py b/src/sqlbuild/compiler/discovery/main/explicit_references_help.py new file mode 100644 index 0000000000..4f2299973a --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/explicit_references_help.py @@ -0,0 +1,13 @@ +"""Entry for the migration help that relaxes explicit references.""" + +from __future__ import annotations + +from sqlbuild.compiler.discovery._helpers.settings.guidance import ( + explicit_references_help as _explicit_references_help, +) + + +def explicit_references_help(*, allowed: str) -> str: + """Show the current `[references] enforce_explicit` value and the exact TOML to relax it.""" + + return _explicit_references_help(allowed=allowed) diff --git a/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py b/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py new file mode 100644 index 0000000000..4980bbcccd --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/microbatch_guidance.py @@ -0,0 +1,14 @@ +"""Entry for the note and help shown when concurrent microbatches are not allowed.""" + +from __future__ import annotations + +from sqlbuild.compiler.discovery._helpers.settings.guidance import ( + microbatch_concurrency_help, + microbatch_concurrency_note, +) + + +def microbatch_guidance() -> tuple[str, str]: + """Return `(note, help)` for `batch_concurrency` above one without the project capability.""" + + return microbatch_concurrency_note(), microbatch_concurrency_help() diff --git a/src/sqlbuild/compiler/discovery/main/sql_analysis_off_guidance.py b/src/sqlbuild/compiler/discovery/main/sql_analysis_off_guidance.py new file mode 100644 index 0000000000..694fb5b000 --- /dev/null +++ b/src/sqlbuild/compiler/discovery/main/sql_analysis_off_guidance.py @@ -0,0 +1,19 @@ +"""Entry for the note and help shown when SQL analysis is turned off by a config file.""" + +from __future__ import annotations + +from sqlbuild.compiler.discovery._helpers.settings.guidance import ( + effective_sql_analysis, + sql_analysis_off_note_and_help, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + + +def sql_analysis_off_guidance( + *, discovered_inputs: DiscoveredProjectInputs, purpose: str +) -> tuple[str, str] | None: + """Return `(note, help)` naming the file that turns SQL analysis off, or None when it is on.""" + + if effective_sql_analysis(discovered_inputs): + return None + return sql_analysis_off_note_and_help(discovered_inputs=discovered_inputs, purpose=purpose) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py b/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py index b72db7f2b2..ec67be90c2 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py @@ -14,8 +14,10 @@ CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.discovery.constants import DEFAULTS_SECTION, PROJECT_CONFIG_FILENAME from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import SourceEntry, TargetConfig @@ -280,6 +282,13 @@ def _validate_required_target_parts( f"{adapter_name} execution requires explicit target {missing_text}. " f"{resource_kind} '{resource_name}' resolved to " f"database={target.database!r} schema={target.schema!r}. " - "Set them in sqlbuild_project.toml defaults, environment config, or model config.", + "Set them in the model config, the environment config, or the project defaults.", code="S101", + help=setting_help( + purpose="to set project-wide defaults", + file_name=PROJECT_CONFIG_FILENAME, + section=DEFAULTS_SECTION, + key=missing_parts[0], + value=f"my_{missing_parts[0]}", + ), ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 5042be73cd..1a0602f3ae 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -26,6 +26,7 @@ CursorInputRoles, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.discovery.main.microbatch_guidance import microbatch_guidance from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map from sqlbuild.compiler.planner._helpers.output.cursor_type_check import ( @@ -79,6 +80,7 @@ build_source_read_map, with_declared_source_reads, ) +from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.changes._model_changes import detect_model_changes from sqlbuild.compiler.planner.main.changes.relation_replacement import ( @@ -498,10 +500,8 @@ def build_plan_entries( ) if configured_concurrent_entries and not project.settings.microbatch_concurrency: names: str = ", ".join(entry.name for entry in configured_concurrent_entries) - raise CompileInputError( - "batch_concurrency > 1 requires settings.microbatch_concurrency = true; " - f"selected model(s): {names}" - ) + note, guidance = microbatch_guidance() + raise CompileInputError(f"{MICROBATCH_CONCURRENCY_MESSAGE}{names}; {note}", help=guidance) if configured_concurrent_entries and not adapter.supports_concurrent_microbatch_dml(): raise CompileInputError( f"adapter '{adapter.adapter_name}' does not support concurrent microbatch " diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index ff28242a79..dd11f37608 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -5,6 +5,11 @@ from sqlbuild.compiler.compile.models import ( CompiledModel, ) +from sqlbuild.compiler.discovery.constants import ( + PROJECT_CONFIG_FILENAME, + QUERY_CHANGE_TRACKING_SETTING_KEY, + SETTINGS_SECTION, +) from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -26,6 +31,7 @@ SchemaColumnSource, WarningSeverity, ) +from sqlbuild.errors.setting_help.main.setting_help import setting_help _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS _OWN_CHANGE_REASONS: dict[ChangeKind, PlanReason] = { @@ -229,9 +235,14 @@ def build_model_warnings( "replay_on_change bounded- e.g. bounded-14d (reprocess just " "that window), or leave it forward-only (existing dates as-is, new " "dates use the new query - the default). If this model's output does " - "not depend on the SQL that changed, this is safe to ignore. Silence " - "these with settings.query_change_tracking = false in " - "sqlbuild_project.toml." + "not depend on the SQL that changed, this is safe to ignore. " + + setting_help( + purpose="To silence these warnings", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=QUERY_CHANGE_TRACKING_SETTING_KEY, + value=False, + ) ), ) ) diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 08f9371d91..b6bcd0e737 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -112,3 +112,6 @@ ), } SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE: str = "source freshness unknown" +MICROBATCH_CONCURRENCY_MESSAGE: str = ( + "batch_concurrency > 1 requires concurrent microbatches; selected model(s): " +) diff --git a/src/sqlbuild/errors/setting_help/__init__.py b/src/sqlbuild/errors/setting_help/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/errors/setting_help/_helpers/__init__.py b/src/sqlbuild/errors/setting_help/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/errors/setting_help/_helpers/text.py b/src/sqlbuild/errors/setting_help/_helpers/text.py new file mode 100644 index 0000000000..cded5445f0 --- /dev/null +++ b/src/sqlbuild/errors/setting_help/_helpers/text.py @@ -0,0 +1,65 @@ +"""Exact TOML and MODEL-header snippets for diagnostics that name a setting.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help.constants import ( + ADDITIONAL_HELP_SEPARATOR, + HELP_CONTINUATION_INDENT, + SETTING_SNIPPET_INDENT, +) +from sqlbuild.errors.setting_help.types import SettingValue + + +def toml_value(value: SettingValue) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def setting_snippet(*, section: str, key: str, value: SettingValue) -> str: + return "\n".join( + ( + f"{SETTING_SNIPPET_INDENT}[{section}]", + f"{SETTING_SNIPPET_INDENT}{key} = {toml_value(value)}", + ) + ) + + +def setting_help( + *, purpose: str, file_name: str, section: str, key: str, value: SettingValue +) -> str: + snippet: str = setting_snippet(section=section, key=key, value=value) + return f"{purpose}, set this in {file_name}:\n{snippet}" + + +def setting_note( + *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None +) -> str: + rendered: str = toml_value(value) + if explicit is None: + return ( + f"the current value is [{section}] {key} = {rendered} (from {file_name} or its default)" + ) + if explicit: + return f"{file_name} sets [{section}] {key} = {rendered}" + return f"{file_name} does not set [{section}] {key}, so it defaults to {rendered}" + + +def model_header_help(*, purpose: str, entry: str, follow_up: str | None) -> str: + lines: list[str] = [ + f"{purpose}, add this to the MODEL header:", + f"{SETTING_SNIPPET_INDENT}MODEL (", + f"{SETTING_SNIPPET_INDENT} {entry},", + f"{SETTING_SNIPPET_INDENT} ...", + f"{SETTING_SNIPPET_INDENT});", + ] + if follow_up is not None: + lines.append(f"{HELP_CONTINUATION_INDENT}{follow_up}") + return "\n".join(lines) + + +def join_helps(helps: tuple[str, ...]) -> str: + return ADDITIONAL_HELP_SEPARATOR.join(helps) diff --git a/src/sqlbuild/errors/setting_help/constants.py b/src/sqlbuild/errors/setting_help/constants.py new file mode 100644 index 0000000000..216f80e559 --- /dev/null +++ b/src/sqlbuild/errors/setting_help/constants.py @@ -0,0 +1,7 @@ +"""Layout of help text that shows an exact setting snippet.""" + +from __future__ import annotations + +SETTING_SNIPPET_INDENT: str = " " * 12 +HELP_CONTINUATION_INDENT: str = " " * 10 +ADDITIONAL_HELP_SEPARATOR: str = "\n = help: " diff --git a/src/sqlbuild/errors/setting_help/main/__init__.py b/src/sqlbuild/errors/setting_help/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/errors/setting_help/main/join_helps.py b/src/sqlbuild/errors/setting_help/main/join_helps.py new file mode 100644 index 0000000000..e63e7214a0 --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/join_helps.py @@ -0,0 +1,11 @@ +"""Public entry for joining several help paragraphs.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import join_helps as _join_helps + + +def join_helps(*helps: str) -> str: + """Join several help paragraphs into one help text, one `= help:` label per paragraph.""" + + return _join_helps(helps) diff --git a/src/sqlbuild/errors/setting_help/main/model_header_help.py b/src/sqlbuild/errors/setting_help/main/model_header_help.py new file mode 100644 index 0000000000..8ad91217dd --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/model_header_help.py @@ -0,0 +1,11 @@ +"""Public entry for help that shows an exact MODEL header entry.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import model_header_help as _model_header_help + + +def model_header_help(*, purpose: str, entry: str, follow_up: str | None = None) -> str: + """Render `, add this to the MODEL header:` followed by the exact header entry.""" + + return _model_header_help(purpose=purpose, entry=entry, follow_up=follow_up) diff --git a/src/sqlbuild/errors/setting_help/main/setting_help.py b/src/sqlbuild/errors/setting_help/main/setting_help.py new file mode 100644 index 0000000000..b802654af8 --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/setting_help.py @@ -0,0 +1,16 @@ +"""Public entry for help that shows the exact setting to write.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import setting_help as _setting_help +from sqlbuild.errors.setting_help.types import SettingValue + + +def setting_help( + *, purpose: str, file_name: str, section: str, key: str, value: SettingValue +) -> str: + """Render `, set this in :` followed by the exact TOML section and key.""" + + return _setting_help( + purpose=purpose, file_name=file_name, section=section, key=key, value=value + ) diff --git a/src/sqlbuild/errors/setting_help/main/setting_note.py b/src/sqlbuild/errors/setting_help/main/setting_note.py new file mode 100644 index 0000000000..3bd64b34fe --- /dev/null +++ b/src/sqlbuild/errors/setting_help/main/setting_note.py @@ -0,0 +1,16 @@ +"""Public entry for the current-value note of a setting.""" + +from __future__ import annotations + +from sqlbuild.errors.setting_help._helpers.text import setting_note as _setting_note +from sqlbuild.errors.setting_help.types import SettingValue + + +def setting_note( + *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None = True +) -> str: + """Render a setting's current value; `explicit=None` when its provenance is unknown.""" + + return _setting_note( + file_name=file_name, section=section, key=key, value=value, explicit=explicit + ) diff --git a/src/sqlbuild/errors/setting_help/types.py b/src/sqlbuild/errors/setting_help/types.py new file mode 100644 index 0000000000..f3060d3fb9 --- /dev/null +++ b/src/sqlbuild/errors/setting_help/types.py @@ -0,0 +1,5 @@ +"""Types for setting help text.""" + +from __future__ import annotations + +type SettingValue = bool | int | str diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index ac5759dc65..14dbccf334 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -9,10 +9,17 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.discovery.constants import ( + PROJECT_CONFIG_FILENAME, + SETTINGS_SECTION, + STAGED_PROMOTION_MODE, + TABLE_PROMOTION_MODE_SETTING_KEY, +) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.final_audits import run_final_model_audits from sqlbuild.executor.run._helpers.execution.hook_phases import ( @@ -382,8 +389,15 @@ def _immediate_lifecycle( phase=ExecutionPhase.TYPE_ENFORCEMENT, error=( f"model '{entry.name}': type enforcement requires staged promotion mode " - f"for runtime column inspection; set table_promotion_mode: staged in " - f"sqlbuild_project.toml settings" + "for runtime column inspection, and [settings] table_promotion_mode is not " + '"staged"; ' + + setting_help( + purpose="to inspect columns before promotion", + file_name=PROJECT_CONFIG_FILENAME, + section=SETTINGS_SECTION, + key=TABLE_PROMOTION_MODE_SETTING_KEY, + value=STAGED_PROMOTION_MODE, + ) ), warnings=warnings, audit_results=audit_results, diff --git a/src/sqlbuild/integrations/dbt/_helpers/config/core.py b/src/sqlbuild/integrations/dbt/_helpers/config/core.py index 4c744daf46..0b7499d74c 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/config/core.py +++ b/src/sqlbuild/integrations/dbt/_helpers/config/core.py @@ -4,6 +4,13 @@ from pathlib import Path +from sqlbuild.compiler.discovery.constants import ( + DBT_PROJECT_DIR_KEY, + DBT_SECTION, + PROJECT_CONFIG_FILENAME, +) +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError from sqlbuild.integrations.dbt.models import DbtCliConfigOverrides, ResolvedDbtConfig from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig @@ -29,7 +36,16 @@ def resolve_dbt_config( if require_project_dir and project_dir is None: raise DbtInteropConfigError( "dbt project directory is not configured", - help="Add [dbt].project_dir to sqlbuild_project.toml or pass --project-dir to sqb dbt.", + help=join_helps( + setting_help( + purpose="to point SQLBuild at the dbt project", + file_name=PROJECT_CONFIG_FILENAME, + section=DBT_SECTION, + key=DBT_PROJECT_DIR_KEY, + value="path/to/dbt_project", + ), + "or pass `--project-dir` to `sqb dbt`", + ), ) raw_profiles_dir: str | None = ( diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 44ca34990e..5b641ee845 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -29,7 +29,7 @@ neutralize_interpolation, sentinel_spans, ) -from sqlbuild.lint.constants import TEMPLATE_INTERPOLATION_START +from sqlbuild.lint.constants import RELATION_IDENTITY_TEMPLATE, TEMPLATE_INTERPOLATION_START from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.models import InterpolationSite, LintBody, LintFileRole from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( @@ -39,6 +39,9 @@ _DEPENDENCY_INTRINSIC_PATTERN: re.Pattern[str] = re.compile( r"^__(?:ref|source)\s*\(", re.IGNORECASE ) +_KEYED_RELATION_PATTERN: re.Pattern[str] = re.compile( + r"^__(ref|source|seed)\s*\(\s*['\"]([^'\"]+)['\"]\s*\)$", re.IGNORECASE +) _CTE_DEFINITION_PATTERN: re.Pattern[str] = re.compile( r'(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)' @@ -225,6 +228,14 @@ def prepare_lint_body( externally_referenced_ctes=externally_referenced_ctes, allows_ceremonial_select=allows_ceremonial_select, allows_dynamic_output_star=allows_dynamic_output_star, + dependency_relations=tuple( + ( + site.sentinel, + RELATION_IDENTITY_TEMPLATE.format(kind=match.group(1).lower(), name=match.group(2)), + ) + for site in sites + if (match := _KEYED_RELATION_PATTERN.match(site.original_text)) is not None + ), ) diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 41ba1582f3..445ad9bbb5 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -4,19 +4,31 @@ import os import tempfile +from collections import Counter from dataclasses import dataclass, field, replace from pathlib import Path from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.lint._helpers.native_format import with_newline_style -from sqlbuild.lint.constants import LINT_ENGINE_NATIVE, VIOLATION_SEVERITY_FAULT +from sqlbuild.lint.constants import ( + LINT_ENGINE_NATIVE, + SAFE_RULE_FIX_CODES, + VIOLATION_SEVERITY_FAULT, + VIOLATION_SEVERITY_WARNING, +) from sqlbuild.lint.exceptions import ProjectCompileError +from sqlbuild.lint.main.compile_facts import compile_facts +from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys +from sqlbuild.lint.main.fix_verdict import fix_verdict from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import ( + CompileFacts, + FixVerdict, FormatChange, LintConfig, LintEdit, @@ -24,19 +36,8 @@ LintViolation, RuleFixResult, ) -from sqlbuild.lint.types import RuleFixStatus - -_SAFE_RULES: frozenset[str] = frozenset( - { - "SQBRSQL002", - "SQBRSQL003", - "SQBRSQL005", - "SQBRSQL006", - "SQBRSQL008", - "SQBRSQL017", - "SQBRSQL025", - } -) +from sqlbuild.lint.types import RelationKeys, RuleFixStatus + _MAX_PASSES: int = 128 _APPLIED: str = "applied" _CONDITIONLESS_JOIN_CODE: str = "SQBRSQL003" @@ -45,6 +46,7 @@ _VERIFICATION_FAILURE: str = ( "Compilation could not prove unchanged output columns, types, dependencies and lineage" ) +_SKIPPED_CODE: str = "format-fix-skipped" @dataclass @@ -57,23 +59,32 @@ class _FixSession: adapter: BaseAdapter inputs: DiscoveredProjectInputs reports: list[RuleFixResult] = field(default_factory=list) - blocked: set[Path] = field(default_factory=set) + blocked: dict[Path, str] = field(default_factory=dict) + skipped: dict[Path, str] = field(default_factory=dict) baseline: CompiledProject | None = None + baseline_facts: CompileFacts | None = None def run(self) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: current: dict[Path, str] = dict(self.original) model_paths: frozenset[Path] = frozenset( model.file_path for model in self.inputs.model_files ) - seen: set[tuple[tuple[Path, str], ...]] = set() + seen: set[tuple[tuple[tuple[Path, str], ...], tuple[Path, ...]]] = set() for _ in range(_MAX_PASSES): - state: tuple[tuple[Path, str], ...] = tuple(sorted(current.items())) + state: tuple[tuple[tuple[Path, str], ...], tuple[Path, ...]] = ( + tuple(sorted(current.items())), + tuple(sorted(self.blocked)), + ) if state in seen: return self._decline(reason="Rule fixes did not converge") seen.add(state) edits, pending, remaining = self._plan_pass(files=current, model_paths=model_paths) if not edits: - return current, tuple(self.reports + remaining), self._blocked_faults() + return ( + current, + (*self.reports, *_not_yet_reported(reports=self.reports, remaining=remaining)), + [*self._blocked_faults(), *self._skipped_warnings()], + ) verified: dict[Path, str] | None = self._verify_pass( current=current, edits=edits, pending=pending ) @@ -88,12 +99,18 @@ def _plan_pass( ) -> tuple[dict[Path, list[LintEdit]], list[RuleFixResult], list[RuleFixResult]]: result: LintRunResult = run_lint( project_dir=self.project_dir, - config=replace(self.config, enabled_native_rules=("SQBRSQL",)), + config=replace( + self.config, + enabled_native_rules=("SQBRSQL",), + relation_keys=self._relation_keys(), + ), value_renderer=self.adapter, discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), source_files=files, selected_paths=frozenset(files), + skip_unexpandable=True, ) + self.skipped.update(result.unexpandable) edits: dict[Path, list[LintEdit]] = {} pending: list[RuleFixResult] = [] remaining: list[RuleFixResult] = [] @@ -119,15 +136,15 @@ def _plan_pass( return edits, pending, remaining def _refusal(self, *, violation: LintViolation, model_paths: frozenset[Path]) -> str | None: - if violation.file_path in self.blocked: - return _VERIFICATION_FAILURE if violation.file_path not in model_paths: return "Compiler-verified Rule fixes currently require a model SQL body" - if violation.fix is not None and violation.code not in _SAFE_RULES: + if violation.fix is not None and violation.code not in SAFE_RULE_FIX_CODES: return "This proposed rewrite is not proven meaning-preserving in this dialect" if violation.code == _CONDITIONLESS_JOIN_CODE and self.config.dialect != _SNOWFLAKE_DIALECT: return "Conditionless JOIN equivalence is only established for Snowflake" - return violation.fix_unavailable_reason + if violation.fix_unavailable_reason is not None or violation.fix is None: + return violation.fix_unavailable_reason + return self.blocked.get(violation.file_path) def _verify_pass( self, @@ -136,74 +153,142 @@ def _verify_pass( edits: dict[Path, list[LintEdit]], pending: list[RuleFixResult], ) -> dict[Path, str] | None: - """Accept every file whose edits verify; revert failing files to their originals.""" - - candidate: dict[Path, str] = _with_edits(files=current, edits=edits) - if self._equivalent_to_baseline(files=candidate): - self.reports.extend(pending) - return candidate - failing: set[Path] = { - path - for path in edits - if not self._equivalent_to_baseline( - files=_with_edits(files=current, edits={path: edits[path]}) + """Attribute failures from one compile of all edits; isolate files when none is to blame.""" + + accepted: dict[Path, list[LintEdit]] = dict(edits) + confirmed: bool = False + while accepted: + verdict: FixVerdict | None = self._verdict( + files=_with_edits(files=self._unblocked(current), edits=accepted) ) - } - if not failing: - return None - self.blocked.update(failing) - reverted: dict[Path, str] = { - path: self.original[path] if path in self.blocked else contents - for path, contents in current.items() - } - accepted: dict[Path, list[LintEdit]] = { - path: path_edits for path, path_edits in edits.items() if path not in failing - } - candidate = _with_edits(files=reverted, edits=accepted) - if not self._equivalent_to_baseline(files=candidate): + if verdict is not None and verdict.verified: + confirmed = True + break + hits: dict[Path, str] = ( + {} if verdict is None else _failing_edits(verdict=verdict, edits=accepted) + ) + if not hits: + accepted = self._isolate(current=current, edits=accepted) + break + for path, reason in hits.items(): + self.blocked[path] = reason + del accepted[path] + candidate: dict[Path, str] = _with_edits(files=self._unblocked(current), edits=accepted) + if accepted and not confirmed and not self._verified(files=candidate): return None self.reports[:] = [ - _refused(report=report) if report.file_path in self.blocked else report + _refused(report=report, reason=self.blocked[report.file_path]) + if report.file_path in self.blocked + else report for report in (*self.reports, *pending) ] return candidate - def _equivalent_to_baseline(self, *, files: dict[Path, str]) -> bool: + def _isolate( + self, *, current: dict[Path, str], edits: dict[Path, list[LintEdit]] + ) -> dict[Path, list[LintEdit]]: + """Check each file's edits alone and keep only the files that verify.""" + + base: dict[Path, str] = self._unblocked(current) + accepted: dict[Path, list[LintEdit]] = {} + for path, path_edits in edits.items(): + verdict: FixVerdict | None = self._verdict( + files=_with_edits(files=base, edits={path: path_edits}) + ) + if verdict is not None and verdict.verified: + accepted[path] = path_edits + else: + self.blocked[path] = ( + _failing_edits(verdict=verdict, edits={path: path_edits}).get( + path, _VERIFICATION_FAILURE + ) + if verdict is not None + else _VERIFICATION_FAILURE + ) + return accepted + + def _unblocked(self, files: dict[Path, str]) -> dict[Path, str]: + return { + path: self.original[path] if path in self.blocked else contents + for path, contents in files.items() + } + + def _relation_keys(self) -> RelationKeys: + """Declared keys from the baseline compile, so determinism proofs match compile.""" + + if self.baseline is None: + try: + self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + except CompileInputError: + return {} + return compiled_relation_keys(self.baseline) + + def _verdict(self, *, files: dict[Path, str]) -> FixVerdict | None: + """Compare one candidate compilation with the baseline; None when it cannot compile.""" + + verdict: FixVerdict | None try: if self.baseline is None: self.baseline = compile_project(discovered_inputs=self.inputs, adapter=self.adapter) + if self.baseline_facts is None: + self.baseline_facts = compile_facts( + project=self.baseline, project_dir=self.project_dir + ) compiled: CompiledProject = compile_project( discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), adapter=self.adapter, ) - valid: bool = _equivalent( - before=self.baseline, - after=compiled, - affected=_affected_models( - project=self.baseline, - changed=frozenset( - path for path, contents in files.items() if contents != self.original[path] - ), - project_dir=self.project_dir, + verdict = fix_verdict( + before=self.baseline_facts, + after=compile_facts(project=compiled, project_dir=self.project_dir), + edited=frozenset( + path.resolve() + for path, contents in files.items() + if contents != self.original[path] ), ) except Exception: - valid = False - return valid + verdict = None + return verdict + + def _verified(self, *, files: dict[Path, str]) -> bool: + verdict: FixVerdict | None = self._verdict(files=files) + return verdict is not None and verdict.verified def _blocked_faults(self) -> list[LintViolation]: - return [_verification_fault(path=path) for path in sorted(self.blocked)] + return [ + _verification_fault(path=path, reason=reason) + for path, reason in sorted(self.blocked.items()) + ] + + def _skipped_warnings(self) -> list[LintViolation]: + return [ + LintViolation( + file_path=path, + line=1, + column=1, + code=_SKIPPED_CODE, + message=f"Rule fixes skipped this file: {reason}", + severity=VIOLATION_SEVERITY_WARNING, + engine=LINT_ENGINE_NATIVE, + remediation="Fix the reported problem, then rerun `sqb format --fix`.", + ) + for path, reason in sorted(self.skipped.items()) + ] def _decline( self, *, reason: str ) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: affected: set[Path] = { report.file_path for report in self.reports if report.status == _APPLIED - } | self.blocked + } | set(self.blocked) return ( self.original, tuple(_refused(report=report, reason=reason) for report in self.reports), - [_verification_fault(path=path, reason=reason) for path in sorted(affected)], + [ + *(_verification_fault(path=path, reason=reason) for path in sorted(affected)), + *self._skipped_warnings(), + ], ) @@ -276,6 +361,15 @@ def plan_rule_fixes( return session.run() +def _failing_edits(*, verdict: FixVerdict, edits: dict[Path, list[LintEdit]]) -> dict[Path, str]: + """Failing files of a verdict among the edited files, keyed as the edits are.""" + + by_resolved: dict[Path, Path] = {path.resolve(): path for path in edits} + return { + by_resolved[path]: reason for path, reason in verdict.failing.items() if path in by_resolved + } + + def _with_edits(*, files: dict[Path, str], edits: dict[Path, list[LintEdit]]) -> dict[Path, str]: candidate: dict[Path, str] = dict(files) for path, path_edits in edits.items(): @@ -286,6 +380,21 @@ def _with_edits(*, files: dict[Path, str], edits: dict[Path, list[LintEdit]]) -> return candidate +def _not_yet_reported( + *, reports: list[RuleFixResult], remaining: list[RuleFixResult] +) -> list[RuleFixResult]: + """Final-pass outcomes, minus those an earlier pass already recorded identically.""" + + recorded: Counter[RuleFixResult] = Counter(reports) + fresh: list[RuleFixResult] = [] + for report in remaining: + if recorded[report] > 0: + recorded[report] -= 1 + else: + fresh.append(report) + return fresh + + def _report(*, violation: LintViolation, status: RuleFixStatus, reason: str) -> RuleFixResult: return RuleFixResult( file_path=violation.file_path, @@ -331,55 +440,3 @@ def _updated_inputs( for model in inputs.model_files ), ) - - -def _affected_models( - *, project: CompiledProject, changed: frozenset[Path], project_dir: Path -) -> frozenset[CompiledObjectKey]: - """Models in changed files and everything downstream; all models if a file is unmatched.""" - - resolved: frozenset[Path] = frozenset(path.resolve() for path in changed) - affected: set[CompiledObjectKey] = { - model.key - for model in project.models - if (project_dir / model.relative_path).resolve() in resolved - } - if len(affected) < len(resolved): - return frozenset(model.key for model in project.models) - grew: bool = True - while grew: - grew = False - for model in project.models: - if model.key not in affected and any(dep in affected for dep in model.deps): - affected.add(model.key) - grew = True - return frozenset(affected) - - -def _equivalent( - *, before: CompiledProject, after: CompiledProject, affected: frozenset[CompiledObjectKey] -) -> bool: - """Affected models must prove unchanged columns, dependencies and lineage.""" - - if any(diagnostic.is_error for diagnostic in (*before.diagnostics, *after.diagnostics)): - return False - if {model.key for model in before.models} != {model.key for model in after.models}: - return False - after_models: dict[CompiledObjectKey, CompiledModel] = { - model.key: model for model in after.models - } - for model in before.models: - if model.key not in affected: - continue - candidate: CompiledModel = after_models[model.key] - if ( - model.inferred_columns is None - or candidate.inferred_columns is None - or model.fast_lineage_columns is None - or candidate.fast_lineage_columns is None - or model.inferred_columns != candidate.inferred_columns - or model.deps != candidate.deps - or tuple(model.fast_lineage_columns) != tuple(candidate.fast_lineage_columns) - ): - return False - return True diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index 21b28cba04..1ec25f0171 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -41,6 +41,7 @@ _NATIVE_LINT_API_VERSION: int = 1 _NEWLINE_CHARACTER: str = "\n" _UNUSED_CTE_CODE: str = "SQBRSQL005" +_LONG_LITERAL_CODE: str = "SQBRSQL044" _PARSE_ERROR_POSITION_PATTERN: re.Pattern[str] = re.compile( r"^Parse error at line (?P\d+), column (?P\d+):" ) @@ -70,6 +71,9 @@ bool, bool, bool, + int, + int, + tuple[tuple[str, tuple[tuple[str, ...], ...]], ...], ] type _NativeResult = dict[str, Any] | NativeLintError @@ -109,6 +113,10 @@ def run_native_sql_lint( payload["allows_dynamic_output_star"] = True if body.allows_empty_fixture_star: payload["allows_empty_fixture_star"] = True + payload["max_literal_length"] = config.max_literal_length + payload["max_ranking_order_by"] = config.max_ranking_order_by + if cache_key[-1]: + payload["relation_keys"] = dict(cache_key[-1]) requests[cache_key] = payload response_cache: dict[_NativeCacheKey, _NativeResult] = _native_responses(requests=requests) @@ -156,6 +164,21 @@ def _native_cache_key(*, body: LintBody, config: LintConfig) -> _NativeCacheKey: body.allows_ceremonial_select, body.allows_dynamic_output_star, body.allows_empty_fixture_star, + config.max_literal_length, + config.max_ranking_order_by, + _body_relation_keys(body=body, config=config), + ) + + +def _body_relation_keys( + *, body: LintBody, config: LintConfig +) -> tuple[tuple[str, tuple[tuple[str, ...], ...]], ...]: + """Declared keys of the relations one body reads, by their neutralized sentinel.""" + + return tuple( + (sentinel, config.relation_keys[name]) + for sentinel, name in body.dependency_relations + if config.relation_keys.get(name) ) @@ -257,6 +280,8 @@ def _authored_violation( raise NativeLintError("native lint diagnostic has an invalid fix refusal reason") mapped: MappedOffset = map_expanded_offset(offset=start, passes=body.passes) absolute_offset: int = body.body_start + mapped.offset + if code == _LONG_LITERAL_CODE and mapped.generated: + return None if code == _UNUSED_CTE_CODE and contents.startswith( _SQLBUILD_HARNESS_CTE_PREFIXES, absolute_offset ): diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 36821aa5a3..9ae2581a3f 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -41,6 +41,22 @@ DEFAULT_MAX_DESCRIPTION_LINES: int = 10 DEFAULT_LINE_WIDTH: int = 100 +DEFAULT_MAX_LITERAL_LENGTH: int = 100 +DEFAULT_MAX_RANKING_ORDER_BY: int = 5 +MAX_LITERAL_LENGTH_THRESHOLD: str = "max_literal_length" +MAX_RANKING_ORDER_BY_THRESHOLD: str = "max_ranking_order_by" +SAFE_RULE_FIX_CODES: frozenset[str] = frozenset( + { + "SQBRSQL002", + "SQBRSQL003", + "SQBRSQL005", + "SQBRSQL006", + "SQBRSQL008", + "SQBRSQL017", + "SQBRSQL025", + "SQBRSQL042", + } +) LINT_DIRECTORY_NAMES: tuple[str, ...] = ( "models", @@ -98,3 +114,25 @@ LINE_FEED: str = "\n" CARRIAGE_RETURN_LINE_FEED: str = "\r\n" TAB_CHARACTER: str = "\t" +UNIQUE_AUDIT_NAME: str = "unique" +UNIQUE_AUDIT_COLUMN_ARGUMENT: str = "column" +UNIQUE_KEY_CONFIG: str = "unique_key" +NOT_NULL_AUDIT_NAME: str = "not_null" +MATERIALIZED_CONFIG: str = "materialized" +INCREMENTAL_MATERIALIZATION: str = "incremental" +INCREMENTAL_MODE_CONFIG: str = "incremental_mode" +MICROBATCH_MODE: str = "microbatch" +INCREMENTAL_STRATEGY_CONFIG: str = "incremental_strategy" +CURSOR_CONFIG: str = "cursor" +KEYED_DELETE_INSERT_STRATEGY: str = "delete_insert" +KEY_PRESERVING_STRATEGIES: frozenset[str] = frozenset({"merge", KEYED_DELETE_INSERT_STRATEGY}) +RELATION_IDENTITY_TEMPLATE: str = "{kind}:{name}" +RELATION_KIND_REF: str = "ref" +RELATION_KIND_SOURCE: str = "source" +RELATION_KIND_SEED: str = "seed" + +FIX_COLUMNS_UNKNOWN: str = "Output columns could not be inferred, so the fix cannot be verified" +FIX_COLUMNS_CHANGED: str = "The fix would change output columns, types or nullability" +FIX_DEPENDENCIES_CHANGED: str = "The fix would change model dependencies" +FIX_LINEAGE_CHANGED: str = "The fix would change column lineage" +FIX_NEW_DIAGNOSTICS: str = "The fix would add compile diagnostics: " diff --git a/src/sqlbuild/lint/main/compile_facts.py b/src/sqlbuild/lint/main/compile_facts.py new file mode 100644 index 0000000000..43a00b0f76 --- /dev/null +++ b/src/sqlbuild/lint/main/compile_facts.py @@ -0,0 +1,58 @@ +"""Index a compilation so two compilations can be compared file by file.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.lint.models import CompileFacts +from sqlbuild.lint.types import DiagnosticIdentity + + +def compile_facts(*, project: CompiledProject, project_dir: Path) -> CompileFacts: + """Index a compilation so two compilations can be compared file by file.""" + + model_paths: dict[CompiledObjectKey, Path] = { + model.key: (project_dir / model.relative_path).resolve() for model in project.models + } + paths_by_name: dict[str, Path] = { + model.name: model_paths[model.key] for model in project.models + } + diagnostics: dict[Path, Counter[DiagnosticIdentity]] = {} + for diagnostic in project.diagnostics: + owner: Path = _owner( + diagnostic=diagnostic, paths_by_name=paths_by_name, project_dir=project_dir + ) + diagnostics.setdefault(owner, Counter())[_identity(diagnostic)] += 1 + return CompileFacts( + models={model.key: model for model in project.models}, + model_paths=model_paths, + diagnostics=diagnostics, + ) + + +def _owner( + *, diagnostic: CompilerDiagnostic, paths_by_name: dict[str, Path], project_dir: Path +) -> Path: + if ( + diagnostic.resource_type == CompiledResourceType.MODEL + and diagnostic.resource_name in paths_by_name + ): + return paths_by_name[diagnostic.resource_name] + path: Path | None = diagnostic.path or ( + diagnostic.location.path if diagnostic.location is not None else None + ) + return (project_dir / path).resolve() if path is not None else project_dir.resolve() + + +def _identity(diagnostic: CompilerDiagnostic) -> DiagnosticIdentity: + """Location-insensitive identity, so moved lines do not count as new diagnostics.""" + + return ( + diagnostic.code, + str(diagnostic.resource_type or ""), + diagnostic.resource_name or "", + diagnostic.message, + ) diff --git a/src/sqlbuild/lint/main/compiled_relation_keys.py b/src/sqlbuild/lint/main/compiled_relation_keys.py new file mode 100644 index 0000000000..6204dc97b4 --- /dev/null +++ b/src/sqlbuild/lint/main/compiled_relation_keys.py @@ -0,0 +1,26 @@ +"""Declared unique keys of a compiled project's relations.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.lint.constants import RELATION_KIND_REF, RELATION_KIND_SEED, RELATION_KIND_SOURCE +from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys +from sqlbuild.lint.types import RelationKeys + + +def compiled_relation_keys(project: CompiledProject) -> RelationKeys: + """Collect declared keys of compiled models, seeds and sources.""" + + return declared_relation_keys( + relations=( + *( + (RELATION_KIND_REF, model.name, model.schema_entry, model.config.values) + for model in project.models + ), + *((RELATION_KIND_SEED, seed.name, seed.schema_entry, {}) for seed in project.seeds), + *( + (RELATION_KIND_SOURCE, source.name, source.source_entry, {}) + for source in project.sources + ), + ) + ) diff --git a/src/sqlbuild/lint/main/declared_relation_keys.py b/src/sqlbuild/lint/main/declared_relation_keys.py new file mode 100644 index 0000000000..f340aa41e8 --- /dev/null +++ b/src/sqlbuild/lint/main/declared_relation_keys.py @@ -0,0 +1,82 @@ +"""Declared unique keys of project relations, for ranking determinism proofs.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +from sqlbuild.lint.constants import ( + CURSOR_CONFIG, + INCREMENTAL_MATERIALIZATION, + INCREMENTAL_MODE_CONFIG, + INCREMENTAL_STRATEGY_CONFIG, + KEY_PRESERVING_STRATEGIES, + KEYED_DELETE_INSERT_STRATEGY, + MATERIALIZED_CONFIG, + MICROBATCH_MODE, + NOT_NULL_AUDIT_NAME, + RELATION_IDENTITY_TEMPLATE, + RELATION_KIND_REF, + RELATION_KIND_SEED, + UNIQUE_AUDIT_COLUMN_ARGUMENT, + UNIQUE_AUDIT_NAME, + UNIQUE_KEY_CONFIG, +) +from sqlbuild.lint.types import RelationKeys + + +def declared_relation_keys( + *, relations: Iterable[tuple[str, str, object, Mapping[str, object]]] +) -> RelationKeys: + """Collect keys that prove one row per key from `(kind, name, entry, config)` relations.""" + + keys: dict[str, list[tuple[str, ...]]] = {} + for kind, name, entry, values in relations: + found: list[tuple[str, ...]] = _trusted_unique_key(values=values) + found.extend((column,) for column in _non_null_unique_columns(entry=entry)) + if not found: + continue + kinds: tuple[str, ...] = ( + (kind, RELATION_KIND_REF) if kind == RELATION_KIND_SEED else (kind,) + ) + for identity_kind in kinds: + identity: str = RELATION_IDENTITY_TEMPLATE.format(kind=identity_kind, name=name) + keys.setdefault(identity, []).extend(found) + return {name: tuple(dict.fromkeys(found)) for name, found in sorted(keys.items())} + + +def _non_null_unique_columns(*, entry: object) -> list[str]: + """Columns with a `unique` audit that are also non-null by audit or `nullable false`.""" + + audited: dict[str, set[str]] = {UNIQUE_AUDIT_NAME: set(), NOT_NULL_AUDIT_NAME: set()} + unique: list[str] = [] + for column in getattr(entry, "columns", None) or (): + names: set[str] = {audit.definition_name for audit in getattr(column, "audits", None) or ()} + if UNIQUE_AUDIT_NAME in names: + unique.append(str(column.name)) + if NOT_NULL_AUDIT_NAME in names or getattr(column, "nullable", None) is False: + audited[NOT_NULL_AUDIT_NAME].add(str(column.name).casefold()) + for audit in getattr(entry, "audits", None) or (): + argument: object = audit.arguments.get(UNIQUE_AUDIT_COLUMN_ARGUMENT) + if audit.definition_name in audited and isinstance(argument, str): + audited[audit.definition_name].add(argument.casefold()) + if audit.definition_name == UNIQUE_AUDIT_NAME: + unique.append(argument) + return [column for column in unique if column.casefold() in audited[NOT_NULL_AUDIT_NAME]] + + +def _trusted_unique_key(*, values: Mapping[str, object]) -> list[tuple[str, ...]]: + """`unique_key` only where the incremental strategy keeps one row per key.""" + + strategy: str = str(values.get(INCREMENTAL_STRATEGY_CONFIG, "")).lower() + trusted: bool = ( + str(values.get(MATERIALIZED_CONFIG, "")).lower() == INCREMENTAL_MATERIALIZATION + and str(values.get(INCREMENTAL_MODE_CONFIG, "")).lower() != MICROBATCH_MODE + and strategy in KEY_PRESERVING_STRATEGIES + and not (strategy == KEYED_DELETE_INSERT_STRATEGY and values.get(CURSOR_CONFIG)) + ) + value: object = values.get(UNIQUE_KEY_CONFIG) if trusted else None + if isinstance(value, str): + return [(value,)] + if isinstance(value, list | tuple) and value and all(isinstance(item, str) for item in value): + return [tuple(str(item) for item in value)] + return [] diff --git a/src/sqlbuild/lint/main/fix_verdict.py b/src/sqlbuild/lint/main/fix_verdict.py new file mode 100644 index 0000000000..dc7d12aaec --- /dev/null +++ b/src/sqlbuild/lint/main/fix_verdict.py @@ -0,0 +1,73 @@ +"""Attribute compile differences between a baseline and a fixed project to edited files.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.lint.constants import ( + FIX_COLUMNS_CHANGED, + FIX_COLUMNS_UNKNOWN, + FIX_DEPENDENCIES_CHANGED, + FIX_LINEAGE_CHANGED, + FIX_NEW_DIAGNOSTICS, +) +from sqlbuild.lint.models import CompileFacts, FixVerdict +from sqlbuild.lint.types import DiagnosticIdentity + + +def fix_verdict( + *, before: CompileFacts, after: CompileFacts, edited: frozenset[Path] +) -> FixVerdict: + """Edited files may lose diagnostics but not gain any; everything else must not change.""" + + if set(before.models) != set(after.models): + return FixVerdict(failing={}, unattributed=True) + failing: dict[Path, str] = {} + unattributed: bool = False + for key, model in before.models.items(): + path: Path = before.model_paths[key] + reason: str | None = _model_change( + before=model, after=after.models[key], require_columns=path in edited + ) + if reason is None: + continue + if path in edited: + failing.setdefault(path, reason) + else: + unattributed = True + empty: Counter[DiagnosticIdentity] = Counter() + for owner in set(before.diagnostics) | set(after.diagnostics): + previous: Counter[DiagnosticIdentity] = before.diagnostics.get(owner, empty) + current: Counter[DiagnosticIdentity] = after.diagnostics.get(owner, empty) + if owner in edited: + added: Counter[DiagnosticIdentity] = current - previous + if added: + codes: str = ", ".join(sorted({identity[0] for identity in added})) + failing.setdefault(owner, FIX_NEW_DIAGNOSTICS + codes) + elif previous != current: + unattributed = True + return FixVerdict(failing=failing, unattributed=unattributed) + + +def _model_change( + *, before: CompiledModel, after: CompiledModel, require_columns: bool +) -> str | None: + """Edited models need known, equal columns; others must not change; absent lineage matches.""" + + if require_columns and (before.inferred_columns is None or after.inferred_columns is None): + return FIX_COLUMNS_UNKNOWN + if before.inferred_columns != after.inferred_columns: + return FIX_COLUMNS_CHANGED + if before.deps != after.deps: + return FIX_DEPENDENCIES_CHANGED + lineage_before: object = ( + None if before.fast_lineage_columns is None else tuple(before.fast_lineage_columns) + ) + lineage_after: object = ( + None if after.fast_lineage_columns is None else tuple(after.fast_lineage_columns) + ) + if lineage_before != lineage_after: + return FIX_LINEAGE_CHANGED + return None diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py index 606e1395a8..e768fa0bf5 100644 --- a/src/sqlbuild/lint/main/run_lint.py +++ b/src/sqlbuild/lint/main/run_lint.py @@ -21,6 +21,7 @@ from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions from sqlbuild.lint.constants import HEADER_KIND_SCENARIO, HEADER_KIND_TEST +from sqlbuild.lint.exceptions import ProjectCompileError from sqlbuild.lint.models import ( HeaderSpan, LintBody, @@ -42,14 +43,16 @@ def run_lint( compiled_expansions: dict[Path, CompiledSqlExpansion] | None = None, expansion_context: SqlExpansionContext | None = None, source_files: dict[Path, str] | None = None, + skip_unexpandable: bool = False, ) -> LintRunResult: - """Lint all DSL files in the project without modifying anything.""" + """Lint all DSL files without modifying anything; optionally skip unexpandable SQL.""" files: dict[Path, str] = collect_project_files( project_dir=project_dir, selected_paths=selected_paths, source_files=source_files ) violations: list[LintViolation] = [] bodies: list[LintBody] = [] + unexpandable: dict[Path, str] = {} context: SqlExpansionContext | None = expansion_context or _expansion_context( project_dir=project_dir, native_enabled=config.native_enabled, @@ -82,20 +85,24 @@ def run_lint( config=config, ) ) - if context is not None: - bodies.extend( - _prepared_bodies( - file_path=file_path, - contents=contents, - headers=headers, - context=context, - dialect=config.dialect, - project_dir=project_dir, - relative_path=relative_path, - dynamic_output_paths=dynamic_output_paths, - compiled_expansions=compiled_expansions, - ) - ) + if context is None: + continue + prepared: tuple[LintBody, ...] | str = _prepared_bodies( + file_path=file_path, + contents=contents, + headers=headers, + context=context, + dialect=config.dialect, + project_dir=project_dir, + relative_path=relative_path, + dynamic_output_paths=dynamic_output_paths, + compiled_expansions=compiled_expansions, + skip_unexpandable=skip_unexpandable, + ) + if isinstance(prepared, str): + unexpandable[file_path] = prepared + else: + bodies.extend(prepared) if bodies and config.native_enabled: native_violations: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( @@ -112,6 +119,7 @@ def run_lint( ), formatted_files=(), source_texts=files, + unexpandable=unexpandable, ) @@ -142,7 +150,10 @@ def _prepared_bodies( relative_path: Path, dynamic_output_paths: frozenset[Path], compiled_expansions: dict[Path, CompiledSqlExpansion] | None, -) -> tuple[LintBody, ...]: + skip_unexpandable: bool, +) -> tuple[LintBody, ...] | str: + """Prepared bodies of one file, or the reason its SQL cannot be expanded when skipping.""" + bodies: list[LintBody] = [] role: LintFileRole = lint_file_role( file_path=file_path, project_dir=project_dir, relative_path=relative_path @@ -160,28 +171,33 @@ def _prepared_bodies( allows_empty_fixture_star: bool = any(header.kind == HEADER_KIND_TEST for header in headers) body_start: int body_end: int - for body_start, body_end in lint_body_ranges( - contents=contents, - headers=headers, - file_path=file_path, - project_dir=project_dir, - role=role, - ): - bodies.append( - replace( - prepare_lint_body( - role=role, - file_path=file_path, - contents=contents, - body_range=(body_start, body_end), - context=context, - dialect=dialect, - external_identifiers=external_identifiers, - allows_ceremonial_select=allows_ceremonial_select, - allows_dynamic_output_star=allows_dynamic_output_star, - compiled_expansion=compiled_expansion, - ), - allows_empty_fixture_star=allows_empty_fixture_star, + try: + for body_start, body_end in lint_body_ranges( + contents=contents, + headers=headers, + file_path=file_path, + project_dir=project_dir, + role=role, + ): + bodies.append( + replace( + prepare_lint_body( + role=role, + file_path=file_path, + contents=contents, + body_range=(body_start, body_end), + context=context, + dialect=dialect, + external_identifiers=external_identifiers, + allows_ceremonial_select=allows_ceremonial_select, + allows_dynamic_output_star=allows_dynamic_output_star, + compiled_expansion=compiled_expansion, + ), + allows_empty_fixture_star=allows_empty_fixture_star, + ) ) - ) + except ProjectCompileError as error: + if not skip_unexpandable: + raise + return str(error).removeprefix(f"{file_path} ") return tuple(bodies) diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index b5e1292392..296ac9ca06 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -2,14 +2,20 @@ from __future__ import annotations +from collections import Counter from collections.abc import Mapping from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.compile.models import ExpansionSpan +from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, ExpansionSpan from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint.constants import VIOLATION_SEVERITY_FAULT, VIOLATION_SEVERITY_WARNING -from sqlbuild.lint.types import LintSeverity, RuleFixStatus +from sqlbuild.lint.constants import ( + DEFAULT_MAX_LITERAL_LENGTH, + DEFAULT_MAX_RANKING_ORDER_BY, + VIOLATION_SEVERITY_FAULT, + VIOLATION_SEVERITY_WARNING, +) +from sqlbuild.lint.types import DiagnosticIdentity, LintSeverity, RuleFixStatus @dataclass(frozen=True) @@ -57,6 +63,7 @@ class LintBody: allows_ceremonial_select: bool = False allows_dynamic_output_star: bool = False allows_empty_fixture_star: bool = False + dependency_relations: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) @@ -127,6 +134,11 @@ class LintConfig: enabled_native_rules: tuple[str, ...] | None = None ignored_native_rules: tuple[str, ...] = () header_rules_enabled: bool = True + max_literal_length: int = DEFAULT_MAX_LITERAL_LENGTH + max_ranking_order_by: int = DEFAULT_MAX_RANKING_ORDER_BY + relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] = field( + default_factory=dict, compare=False + ) @dataclass(frozen=True) @@ -139,6 +151,7 @@ class LintRunResult: format_changes: tuple[FormatChange, ...] = () rule_fixes: tuple[RuleFixResult, ...] = () source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) + unexpandable: Mapping[Path, str] = field(default_factory=dict) @property def faults(self) -> tuple[LintViolation, ...]: @@ -159,3 +172,24 @@ def warnings(self) -> tuple[LintViolation, ...]: for violation in self.violations if violation.severity == VIOLATION_SEVERITY_WARNING ) + + +@dataclass(frozen=True) +class CompileFacts: + """Per-model outputs and per-owner diagnostics of one compilation.""" + + models: dict[CompiledObjectKey, CompiledModel] + model_paths: dict[CompiledObjectKey, Path] + diagnostics: dict[Path, Counter[DiagnosticIdentity]] + + +@dataclass(frozen=True) +class FixVerdict: + """Edited files that fail verification, and whether any change is unattributable.""" + + failing: dict[Path, str] + unattributed: bool + + @property + def verified(self) -> bool: + return not self.failing and not self.unattributed diff --git a/src/sqlbuild/lint/types.py b/src/sqlbuild/lint/types.py index 298cbb8b50..6909aadd68 100644 --- a/src/sqlbuild/lint/types.py +++ b/src/sqlbuild/lint/types.py @@ -14,3 +14,7 @@ class LintSeverity(StrEnum): type RuleFixStatus = Literal["applied", "refused", "unavailable"] + +type RelationKeys = dict[str, tuple[tuple[str, ...], ...]] + +type DiagnosticIdentity = tuple[str, str, str, str] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index c2d18d301f..57555ec24c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -209,7 +209,24 @@ def finalize_native_findings( raise RulesError(str(error)) from error if not isinstance(payload, list): raise RulesError("native rules engine returned invalid finalized findings") - return tuple(decode_rule_finding(value) for value in payload) + fixable: frozenset[tuple[str, str, int, int, str]] = frozenset( + _finding_identity(finding) for finding in findings if finding.fixable + ) + finalized: tuple[Finding, ...] = tuple(decode_rule_finding(value) for value in payload) + return tuple( + replace(finding, fixable=True) if _finding_identity(finding) in fixable else finding + for finding in finalized + ) + + +def _finding_identity(finding: Finding) -> tuple[str, str, int, int, str]: + return ( + finding.code, + finding.path.as_posix(), + finding.line, + finding.column, + finding.message, + ) def load_native_config(project_dir: Path) -> dict[str, object]: @@ -822,6 +839,7 @@ def decode_rule_finding(value: object) -> Finding: message=message, remediation=remediation, unevaluated=payload.get("unevaluated") is True, + fixable=payload.get("fixable") is True, ) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py index 134d8c9227..2ac99ed591 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py @@ -5,6 +5,7 @@ import json from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines +from sqlbuild.rule_engine.constants import RULE_FIX_AVAILABLE_NOTE from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult _MINIMUM_RULES_CODES: frozenset[str] = frozenset({"SQBRTEST201", "SQBRTEST202"}) @@ -25,6 +26,8 @@ def format_result_text(*, result: RulesResult) -> str: ) if finding.affected_rules: blocks.append(f" Affected Rules: {', '.join(finding.affected_rules)}") + if finding.fixable: + blocks.append(f" {RULE_FIX_AVAILABLE_NOTE}") blocks.append(f"Found {len(result.findings)} Rules findings") if result.unevaluated_resources: blocks.append( @@ -50,6 +53,7 @@ def format_result_json(*, result: RulesResult) -> str: if finding.affected_rules else {} ), + **({"fixable": True} if finding.fixable else {}), } ) payload: dict[str, object] = { diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 89b37e8877..3bacf498ef 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -118,6 +118,7 @@ def _decode_inputs(payload: dict[str, Any]) -> tuple[CompiledProject, RulesConfi def _finding_payload(finding: Finding) -> dict[str, object]: payload: dict[str, object] = asdict(finding) payload.pop("affected_rules", None) + payload.pop("fixable", None) payload["path"] = finding.path.as_posix() return payload diff --git a/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py b/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py new file mode 100644 index 0000000000..7212cade71 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/literal_duplicates.py @@ -0,0 +1,115 @@ +"""Point repeated long literals at one shared constant in the nearest common folder.""" + +from __future__ import annotations + +import os +from dataclasses import replace +from pathlib import Path, PurePosixPath + +from sqlbuild.rule_engine.models import Finding + +_LONG_LITERAL_CODE: str = "SQBRSQL044" +_QUOTE: str = "'" +_LISTED_PATHS: int = 5 +_MIN_SHARED_FILES: int = 2 +_MIN_SCOPED_FOLDER_DEPTH: int = 2 +_DECLARATION_GROUP: str = "_sqlbuild" +_LOCAL_CONSTANTS: str = "_sqlbuild/_constants/" +_INHERITED_CONSTANTS: str = "_sqlbuild/constants/" +_PROJECT_CONSTANTS: str = "constants/" + + +def with_duplicate_literal_hints( + *, findings: tuple[Finding, ...], project_dir: Path +) -> tuple[Finding, ...]: + """Prefix SQBRSQL044 help with the other files that repeat the identical literal.""" + + texts: dict[Path, list[str]] = {} + literals: dict[int, str] = {} + for index, finding in enumerate(findings): + if finding.code != _LONG_LITERAL_CODE or finding.unevaluated: + continue + literal: str | None = _literal_at( + lines=_lines(texts=texts, path=project_dir / finding.path), + line=finding.line, + column=finding.column, + ) + if literal is not None: + literals[index] = literal + paths_by_literal: dict[str, set[PurePosixPath]] = {} + for index, literal in literals.items(): + paths_by_literal.setdefault(literal, set()).add(PurePosixPath(findings[index].path)) + hinted: list[Finding] = list(findings) + for index, literal in literals.items(): + paths: set[PurePosixPath] = paths_by_literal[literal] + if len(paths) < _MIN_SHARED_FILES: + continue + own: PurePosixPath = PurePosixPath(findings[index].path) + hint: str = _hint(others=sorted(paths - {own}), every=paths) + hinted[index] = replace( + findings[index], remediation=f"{hint} {findings[index].remediation}" + ) + return tuple(hinted) + + +def _lines(*, texts: dict[Path, list[str]], path: Path) -> list[str]: + if path not in texts: + try: + texts[path] = path.read_text(encoding="utf-8").splitlines() + except OSError: + texts[path] = [] + return texts[path] + + +def _literal_at(*, lines: list[str], line: int, column: int) -> str | None: + """The single-quoted literal starting at a 1-based position, quotes included.""" + + if not 1 <= line <= len(lines): + return None + text: str = lines[line - 1] + start: int = column - 1 + if start < 0 or text[start : start + 1] != _QUOTE: + return None + position: int = start + 1 + while position < len(text): + if text[position] == _QUOTE: + if text[position + 1 : position + 2] == _QUOTE: + position += 2 + continue + return text[start : position + 1] + position += 1 + return None + + +def _hint(*, others: list[PurePosixPath], every: set[PurePosixPath]) -> str: + listed: str = ", ".join(path.as_posix() for path in others[:_LISTED_PATHS]) + more: str = f" and {len(others) - _LISTED_PATHS} more" if len(others) > _LISTED_PATHS else "" + owners: set[PurePosixPath] = {_owner(path) for path in every} + common: PurePosixPath = _common_folder(owners) + if len(common.parts) < _MIN_SCOPED_FOLDER_DEPTH: + placement: str = ( + f"no folder below the project roots holds every use, so declare it once in " + f"top-level `{_PROJECT_CONSTANTS}`" + ) + else: + directory: str = _LOCAL_CONSTANTS if owners == {common} else _INHERITED_CONSTANTS + placement = ( + f"their nearest common folder is `{common.as_posix()}`, so declare it once in " + f"`{common.as_posix()}/{directory}`" + ) + return ( + f"The identical literal also appears in {listed}{more}; {placement} " + "and reference it with `@const` in each file." + ) + + +def _owner(path: PurePosixPath) -> PurePosixPath: + """The folder whose declarations a file sees: above `_sqlbuild/`, else its own folder.""" + + if _DECLARATION_GROUP in path.parts: + return PurePosixPath(*path.parts[: path.parts.index(_DECLARATION_GROUP)]) + return path.parent + + +def _common_folder(folders: set[PurePosixPath]) -> PurePosixPath: + return PurePosixPath(os.path.commonpath([folder.as_posix() for folder in folders])) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index c42aa7120d..962190784c 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -25,12 +25,25 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.constants import TEMPLATE_INTERPOLATION_START +from sqlbuild.lint.constants import ( + DEFAULT_MAX_LITERAL_LENGTH, + DEFAULT_MAX_RANKING_ORDER_BY, + MAX_LITERAL_LENGTH_THRESHOLD, + MAX_RANKING_ORDER_BY_THRESHOLD, + RELATION_KIND_REF, + RELATION_KIND_SEED, + RELATION_KIND_SOURCE, + SAFE_RULE_FIX_CODES, + TEMPLATE_INTERPOLATION_START, +) from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.main.build_expansion_context import build_expansion_context from sqlbuild.lint.main.collect_project_files import collect_project_files +from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys +from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation +from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules @@ -42,6 +55,7 @@ rules_cache_exists, ) from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings +from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config @@ -56,7 +70,7 @@ SqlExpansionReuse, ) -_SQL_RULE_CACHE_VERSION: str = "sql-rules-v3" +_SQL_RULE_CACHE_VERSION: str = "sql-rules-v5" _SQLBUILD_VERSION: str = version("sqlbuild") _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -124,10 +138,16 @@ def evaluate_rules( select=tuple(rule.code for rule in selected), ignore=(), ) + lint_config: LintConfig = sql_lint_config( + dialect=dialect, + codes=_sql_rule_codes(native_rules), + thresholds=effective_config.thresholds, + relation_keys=compiled_relation_keys(graph.project), + ) sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( rules=native_rules, project=selected_project, - dialect=dialect, + dialect=_lint_identity(lint_config), project_dir=resolved_project_dir, collect_files=prepared_sql is None and model_paths is None, ) @@ -148,7 +168,7 @@ def evaluate_rules( project_dir=resolved_project_dir, discovered_inputs=discovered_inputs, project=selected_project, - dialect=dialect, + lint_config=lint_config, selected_model_paths=model_paths, cache_enabled=effective_config.cache.enabled, prepared_sql=prepared_sql, @@ -246,8 +266,36 @@ def prepare_sql_rules( future=executor.submit( _prepare_sql_lint, project_dir=project_dir, - config=LintConfig( - dialect=dialect, enabled_native_rules=codes, header_rules_enabled=False + config=sql_lint_config( + dialect=dialect, + codes=codes, + thresholds=config.thresholds, + relation_keys=declared_relation_keys( + relations=( + *( + ( + RELATION_KIND_REF, + model.model_file.file_path.stem, + model.schema_entry, + model.config.values, + ) + for model in inputs.model_inputs + ), + *( + (RELATION_KIND_SEED, seed.schema_entry.name, seed.schema_entry, {}) + for seed in inputs.seed_inputs + ), + *( + ( + RELATION_KIND_SOURCE, + source.source_entry.name, + source.source_entry, + {}, + ) + for source in inputs.source_inputs + ), + ) + ), ), discovered_inputs=inputs.discovered_inputs, static_declaration_scope=inputs.declaration_scope, @@ -406,7 +454,7 @@ def _run_sql_rules( project_dir: Path, discovered_inputs: DiscoveredProjectInputs, project: CompiledProject, - dialect: str, + lint_config: LintConfig, selected_model_paths: frozenset[str] | None, cache_enabled: bool, prepared_sql: PreparedSqlLint | None = None, @@ -423,7 +471,9 @@ def _run_sql_rules( identities: dict[str, str] = ( prepared_inputs.identities if prepared_inputs is not None - else _sql_model_rule_identities(rules=rules, project=project, dialect=dialect) + else _sql_model_rule_identities( + rules=rules, project=project, dialect=_lint_identity(lint_config) + ) ) findings: list[Finding] = [] misses: set[str] = set() @@ -462,7 +512,7 @@ def _run_sql_rules( relative_path=relative_path, contents=contents, codes=codes, - dialect=dialect, + dialect=_lint_identity(lint_config), sql_expansions=project.sql_expansions, ) cached_file: tuple[Finding, ...] | None = ( @@ -479,9 +529,7 @@ def _run_sql_rules( result: LintRunResult | None = ( _run_prepared_lint( project_dir=project_dir, - config=LintConfig( - dialect=dialect, enabled_native_rules=codes, header_rules_enabled=False - ), + config=lint_config, selected_paths=frozenset(selected_paths), discovered_inputs=discovered_inputs, compiled_expansions=project.sql_expansions, @@ -502,9 +550,17 @@ def _run_sql_rules( evaluated: list[Finding] = [] for violation in () if result is None else result.violations: if violation.code in selected_codes: - evaluated.append(_lint_finding(violation=violation, project_dir=project_dir)) + evaluated.append( + _lint_finding( + violation=violation, + project_dir=project_dir, + fixable_paths=frozenset(models_by_path), + ) + ) elif violation.code == NativeLintError.code: - finding: Finding = _lint_finding(violation=violation, project_dir=project_dir) + finding: Finding = _lint_finding( + violation=violation, project_dir=project_dir, fixable_paths=frozenset() + ) evaluated.extend( replace( finding, @@ -532,7 +588,7 @@ def _run_sql_rules( } _write_sql_rule_cache(project_dir=project_dir, bucket=bucket) return _SqlRulesEvaluation( - findings=tuple(findings), + findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), cache_hits=len(identities) - len(misses), cache_misses=len(misses), ) @@ -558,7 +614,11 @@ def _lint_expansion_context( ) -def _lint_finding(*, violation: LintViolation, project_dir: Path) -> Finding: +def _lint_finding( + *, violation: LintViolation, project_dir: Path, fixable_paths: frozenset[str] +) -> Finding: + """Convert one lint violation; fixable when `format --fix` would attempt its edit.""" + path: Path = violation.file_path if path.is_absolute() and path.is_relative_to(project_dir.resolve()): path = path.relative_to(project_dir.resolve()) @@ -569,6 +629,44 @@ def _lint_finding(*, violation: LintViolation, project_dir: Path) -> Finding: column=violation.column, message=violation.message, remediation=violation.remediation or "Update the authored SQL to satisfy this rule.", + fixable=violation.fix is not None + and violation.code in SAFE_RULE_FIX_CODES + and path.as_posix() in fixable_paths, + ) + + +def sql_lint_config( + *, + dialect: str, + codes: tuple[str, ...], + thresholds: dict[str, int], + relation_keys: RelationKeys, +) -> LintConfig: + """Build the native SQL lint configuration that SQL Rules run with.""" + + return LintConfig( + dialect=dialect, + enabled_native_rules=codes, + header_rules_enabled=False, + max_literal_length=thresholds.get(MAX_LITERAL_LENGTH_THRESHOLD, DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by=thresholds.get( + MAX_RANKING_ORDER_BY_THRESHOLD, DEFAULT_MAX_RANKING_ORDER_BY + ), + relation_keys=relation_keys, + ) + + +def _lint_identity(config: LintConfig) -> str: + """Cache identity of everything besides SQL text that changes SQL Rule findings.""" + + return json.dumps( + [ + config.dialect, + config.max_literal_length, + config.max_ranking_order_by, + sorted(config.relation_keys.items()), + ], + separators=(",", ":"), ) @@ -689,6 +787,7 @@ def _finding_cache_payload(finding: Finding) -> dict[str, object]: "column": finding.column, "message": finding.message, "remediation": finding.remediation, + "fixable": finding.fixable, } diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index ba04b7294c..f2685811ee 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -120,3 +120,6 @@ MAX_ENUM_CONTAINER_FILES: 10, MIN_SHARED_CONTAINER_PREFIX_FILES: 2, } +RULE_FIX_AVAILABLE_NOTE: str = ( + "Fix available: `sqb format --fix` applies it after compiler verification" +) diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 9a78f61444..9c0b57c0d0 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -220,6 +220,7 @@ class Finding: remediation: str unevaluated: bool = False affected_rules: tuple[str, ...] = () + fixable: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 1643d02f19..853a180eb0 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -157,6 +157,7 @@ class SettingsConfig: """Global feature toggles.""" sql_analysis: bool = True + require_sql_analysis: bool = False query_change_tracking: bool = True column_contract_mode: ColumnContractMode = ColumnContractMode.IMPLICIT concurrency: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py index 3c937564e8..9510a9d06b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_concurrent_microbatch_build.py @@ -83,9 +83,11 @@ def test_given_project_capability_disabled_when_batch_concurrency_exceeds_one_th ) assert result.returncode == test_case.expected_exit_code - assert "batch_concurrency > 1 requires settings.microbatch_concurrency = true" in ( - result.stdout + result.stderr - ) + assert ( + "= help: to run microbatches concurrently, set this in sqlbuild_project.toml:\n" + " [settings]\n" + " microbatch_concurrency = true" + ) in (result.stdout + result.stderr) @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py index 58e740eba6..a8fd48689f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_explicit_references.py @@ -366,7 +366,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an "orders_base()", "--> models/sales/_sqlbuild/_macros/unions.py", '@orders_base(__ref("stg_orders_eu"))', - "[references] enforce_explicit = false", + "[references]\n enforce_explicit = false", ), ), ExplicitReferenceFailureE2ETestCase( @@ -386,7 +386,7 @@ def test_given_typed_macro_references_and_hook_reads_when_building_then_graph_an expected_output_fragments=( "error[P008]: task:export_orders names model:stg_customers as 'stg_customers'", "--> python/tasks/export.py:7", - "[references] enforce_explicit = false", + "[references]\n enforce_explicit = false", ), ), ExplicitReferenceFailureE2ETestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py index 88d0926084..1e2db0e347 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py @@ -51,7 +51,10 @@ expected_stderr_fragments=( "Polyglot could not parse model 'orders' pre_hooks[0] inline_sql(\"...\")", "Invalid expression / Unexpected token", - "settings.sql_analysis: false", + " = help: if this SQL is valid for your warehouse, skip SQL analysis for this " + "model, add this to the MODEL header:\n" + " MODEL (\n" + " sql_analysis false,", ), ), CliFailureBuildE2ETestCase( @@ -171,8 +174,9 @@ expected_exit_code=1, expected_stderr_fragments=( "SQL syntax error in source expression 'raw_orders'", - "To disable project-wide, set `settings.sql_analysis: false`", - "To skip for this run, use `--no-sql-analysis`.", + "set this in sqlbuild_project.toml:\n [settings]\n" + " sql_analysis = false", + " = help: to skip SQL analysis for one run, use `--no-sql-analysis`", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index e7e5e9d360..8c798b12bb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -404,3 +404,44 @@ class ModelHeaderKeyCompileCase: description: str repo_files: dict[str, str] expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ResourceSqlValidationCase: + description: str + files: tuple[tuple[str, str], ...] + expected_diagnostics: tuple[tuple[str, str, str, int], ...] + + +@dataclass(frozen=True) +class ResourceSqlOptOutCase: + description: str + files: tuple[tuple[str, str], ...] + flags: tuple[str, ...] + expected_diagnostics: tuple[tuple[str, str, int], ...] + expected_returncode: int = 0 + + +@dataclass(frozen=True) +class ResourceSqlHelpCase: + description: str + files: tuple[tuple[str, str], ...] + expected_diagnostics: tuple[tuple[str, str, str | None], ...] + + +@dataclass(frozen=True) +class TypeFindingLocationCase: + description: str + adapter: str + query_sql: str + expected_diagnostics: tuple[tuple[str, str, int, int], ...] + + +@dataclass(frozen=True) +class RequireSqlAnalysisCase: + description: str + repo_files: tuple[tuple[str, str], ...] + command: tuple[str, ...] + expected_returncode: int + expected_diagnostics: tuple[tuple[str, str, int], ...] + expected_text_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 237943741e..bd7b20fb3c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2515,3 +2515,86 @@ def prepare_compile_cache_invalidation_project(*, project_dir: Path) -> None: "[settings]", '[vars]\nquantity_multiplier = "2"\n\n[settings]', ) + + +RESOURCE_SQL_MODELS: str = "models/staging" +RESOURCE_SQL_ORDERS: str = f"{RESOURCE_SQL_MODELS}/orders.sql" +RESOURCE_SQL_SINGULAR_AUDIT: str = ( + f"{RESOURCE_SQL_MODELS}/_sqlbuild/audits/singular/orders_have_customers.sql" +) +RESOURCE_SQL_GENERIC_AUDIT: str = ( + f"{RESOURCE_SQL_MODELS}/_sqlbuild/_audits/generic/status_is_known.sql" +) +RESOURCE_SQL_RECENT_ROWS_AUDIT: str = ( + f"{RESOURCE_SQL_MODELS}/_sqlbuild/_audits/generic/recent_rows.sql" +) +RESOURCE_SQL_TEST: str = "tests/unit/test_orders.sql" +RESOURCE_SQL_NAMED_HOOK: str = f"{RESOURCE_SQL_MODELS}/_sqlbuild/_hooks/sql/record_orders.sql" +_RESOURCE_SQL_BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n' + ), + f"{RESOURCE_SQL_MODELS}/customers.sql": ( + 'MODEL (description "Customers per order");\n' + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(7 AS INTEGER) AS customer_id\n" + ), +} + + +def resource_sql_orders_model(options: str = "") -> tuple[str, str]: + """Return the orders model file whose header carries the given extra options.""" + + return ( + RESOURCE_SQL_ORDERS, + f'MODEL (description "Orders with their status"{options});\n' + 'WITH customers AS (SELECT * FROM __ref("customers"))\n' + "SELECT CAST(c.order_id AS INTEGER) AS order_id, CAST('open' AS VARCHAR) AS status\n" + "FROM customers AS c\n", + ) + + +def resource_sql_project_files(files: tuple[tuple[str, str], ...]) -> dict[str, str]: + """Return the shared orders project plus case-specific files.""" + + return {**_RESOURCE_SQL_BASE_FILES, **dict(files)} + + +REQUIRE_SQL_ANALYSIS_PROJECT: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[settings]\nrequire_sql_analysis = true\n' +) +OPTIONAL_SQL_ANALYSIS_PROJECT: str = 'name = "orders"\nadapter = "duckdb"\n' +PARSEABLE_OPT_OUT_MODEL: str = ( + 'MODEL (\n description "Order flags",\n sql_analysis false\n);\n' + "SELECT 1 = 'pending' AS is_pending, 2 = 'shipped' AS is_shipped\n" +) +UNPARSEABLE_OPT_OUT_MODEL: str = ( + 'MODEL (description "Order lookup", sql_analysis false);\nSELECT order_id FROM orders WHERE\n' +) +PATH_DEFAULT_MODEL: str = ( + 'MODEL (description "Order epochs");\n' + "SELECT CAST(TIMESTAMP '2026-04-01' AS INTEGER) AS ordered_epoch\n" +) +OPT_OUT_HELP: str = ( + "= help: to allow `sql_analysis false` on any model, SQL test or audit, set this in " + "sqlbuild_project.toml:\n" + " [settings]\n" + " require_sql_analysis = false" +) + + +def require_sql_analysis_output( + result: subprocess.CompletedProcess[str], +) -> tuple[tuple[tuple[str, str, int], ...], str]: + """Return `(code, path, line)` per JSON diagnostic and all JSON help, note and stderr text.""" + + payload: dict[str, Any] = json.loads(result.stdout or '{"diagnostics": []}') + diagnostics: list[dict[str, Any]] = payload["diagnostics"] + parts: list[str] = [result.stderr] + for item in diagnostics: + parts.append(str(item.get("help"))) + parts.extend(item.get("notes", ())) + text: str = "\n".join(parts) + return ( + tuple((item["code"], item["path"], item["line"]) for item in diagnostics), + text, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py new file mode 100644 index 0000000000..c24177f4b4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py @@ -0,0 +1,180 @@ +"""E2E coverage of `[settings] require_sql_analysis` and its exact-TOML error help.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import RequireSqlAnalysisCase +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + OPT_OUT_HELP, + OPTIONAL_SQL_ANALYSIS_PROJECT, + PARSEABLE_OPT_OUT_MODEL, + PATH_DEFAULT_MODEL, + REQUIRE_SQL_ANALYSIS_PROJECT, + UNPARSEABLE_OPT_OUT_MODEL, + require_sql_analysis_output, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_COMPILE: tuple[str, ...] = ("--no-color", "compile", "--json", "--no-cache") + + +@pytest.mark.parametrize( + "test_case", + ( + RequireSqlAnalysisCase( + description="a header opt-out on parseable SQL is an error listing its findings", + repo_files=( + ("sqlbuild_project.toml", REQUIRE_SQL_ANALYSIS_PROJECT), + ("models/order_flags.sql", PARSEABLE_OPT_OUT_MODEL), + ), + command=_COMPILE, + expected_returncode=1, + expected_diagnostics=(("P009", "models/order_flags.sql", 3),), + expected_text_fragments=( + "sqlbuild_project.toml sets [settings] require_sql_analysis = true, which only " + "allows `sql_analysis false` on SQL that SQLBuild cannot parse; this model " + "parses successfully.", + "remove `sql_analysis false` and fix the findings it was hiding: " + "2 type mismatches (run `sqb compile` to see them)", + OPT_OUT_HELP, + ), + ), + RequireSqlAnalysisCase( + description="a path_defaults opt-out is reported at its project file entry", + repo_files=( + ( + "sqlbuild_project.toml", + REQUIRE_SQL_ANALYSIS_PROJECT + + '\n[path_defaults."marts"]\nsql_analysis = false\n', + ), + ("models/marts/order_epochs.sql", PATH_DEFAULT_MODEL), + ), + command=_COMPILE, + expected_returncode=1, + expected_diagnostics=(("P009", "sqlbuild_project.toml", 8),), + expected_text_fragments=( + "remove `sql_analysis = false` from this [path_defaults] entry and fix the " + "findings it was hiding: 1 type mismatch", + OPT_OUT_HELP, + ), + ), + RequireSqlAnalysisCase( + description="an opt-out on SQL the parser rejects is accepted", + repo_files=( + ("sqlbuild_project.toml", REQUIRE_SQL_ANALYSIS_PROJECT), + ("models/order_lookup.sql", UNPARSEABLE_OPT_OUT_MODEL), + ), + command=_COMPILE, + expected_returncode=0, + expected_diagnostics=(), + expected_text_fragments=(), + ), + RequireSqlAnalysisCase( + description="without the setting a parseable opt-out keeps today's behaviour", + repo_files=( + ("sqlbuild_project.toml", OPTIONAL_SQL_ANALYSIS_PROJECT), + ("models/order_flags.sql", PARSEABLE_OPT_OUT_MODEL), + ), + command=_COMPILE, + expected_returncode=0, + expected_diagnostics=(), + expected_text_fragments=(), + ), + RequireSqlAnalysisCase( + description="a single-run --no-sql-analysis still works", + repo_files=( + ("sqlbuild_project.toml", REQUIRE_SQL_ANALYSIS_PROJECT), + ("models/order_flags.sql", PARSEABLE_OPT_OUT_MODEL), + ), + command=("--no-color", "compile", "--json", "--no-cache", "--no-sql-analysis"), + expected_returncode=0, + expected_diagnostics=(), + expected_text_fragments=(), + ), + RequireSqlAnalysisCase( + description="a local project-wide opt-out is rejected", + repo_files=( + ("sqlbuild_project.toml", REQUIRE_SQL_ANALYSIS_PROJECT), + ("sqlbuild_local.toml", "[settings]\nsql_analysis = false\n"), + ("models/order_epochs.sql", PATH_DEFAULT_MODEL), + ), + command=_COMPILE, + expected_returncode=1, + expected_diagnostics=(), + expected_text_fragments=( + "error[D001]:", + "turns SQL analysis off with [settings] sql_analysis = false, which this project " + "does not allow; sqlbuild_project.toml sets [settings] require_sql_analysis = true", + "= help: to allow turning SQL analysis off locally, set this in " + "sqlbuild_project.toml:\n [settings]\n" + " require_sql_analysis = false", + ), + ), + RequireSqlAnalysisCase( + description="require_sql_analysis cannot be set locally", + repo_files=( + ("sqlbuild_project.toml", REQUIRE_SQL_ANALYSIS_PROJECT), + ("sqlbuild_local.toml", "[settings]\nrequire_sql_analysis = false\n"), + ("models/order_epochs.sql", PATH_DEFAULT_MODEL), + ), + command=_COMPILE, + expected_returncode=1, + expected_diagnostics=(), + expected_text_fragments=( + "cannot set [settings] require_sql_analysis; it is a shared project policy that " + "only sqlbuild_project.toml can set", + "= help: remove require_sql_analysis from sqlbuild_local.toml", + ), + ), + RequireSqlAnalysisCase( + description="an unparseable model without an opt-out shows the exact MODEL header", + repo_files=( + ("sqlbuild_project.toml", OPTIONAL_SQL_ANALYSIS_PROJECT), + ( + "models/order_lookup.sql", + 'MODEL (description "Order lookup");\nSELECT order_id FROM orders WHERE\n', + ), + ), + command=_COMPILE, + expected_returncode=1, + expected_diagnostics=(), + expected_text_fragments=( + "error[P001]: SQL syntax error in model 'order_lookup'", + "= help: if this SQL is valid for your warehouse, skip SQL analysis for this " + "model, add this to the MODEL header:\n" + " MODEL (\n" + " sql_analysis false,\n" + " ...\n" + " );\n" + " and please report the SQL so the parser can support it", + "= help: SQL analysis is on for this project; to turn it off for every model, " + "set this in sqlbuild_project.toml:\n [settings]\n" + " sql_analysis = false", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_analysis_opt_outs_when_compiling_then_policy_and_help_match( + test_case: RequireSqlAnalysisCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders", repo_files=dict(test_case.repo_files) + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=test_case.command + ) + + diagnostics, text = require_sql_analysis_output(result) + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert diagnostics == test_case.expected_diagnostics + assert all(fragment in text for fragment in test_case.expected_text_fragments), text + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_resource_sql_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_resource_sql_validation.py new file mode 100644 index 0000000000..a4728990ce --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_resource_sql_validation.py @@ -0,0 +1,558 @@ +"""E2E coverage of compile-time SQL checks on audits, SQL tests and SQL hooks.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + ResourceSqlHelpCase, + ResourceSqlOptOutCase, + ResourceSqlValidationCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + RESOURCE_SQL_GENERIC_AUDIT, + RESOURCE_SQL_MODELS, + RESOURCE_SQL_NAMED_HOOK, + RESOURCE_SQL_ORDERS, + RESOURCE_SQL_RECENT_ROWS_AUDIT, + RESOURCE_SQL_SINGULAR_AUDIT, + RESOURCE_SQL_TEST, + resource_sql_orders_model, + resource_sql_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_REQUIRED_ANALYSIS_TOML: tuple[str, str] = ( + "sqlbuild_project.toml", + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n\n' + "[settings]\nrequire_sql_analysis = true\n", +) +_FINDING_AUDIT_BODY: str = ( + 'SELECT o.order_id\nFROM __ref("orders") AS o\n' + 'JOIN __ref("customers") AS c ON o.order_id = c.order_id\n' + "WHERE STARTSWITH(o.status, 'x')\n" +) +_FINDING_TEST_BODY: str = ( + "WITH __ref__customers AS (SELECT 1 AS order_id, 7 AS customer_id),\n" + "__expected__orders AS (SELECT 1 AS order_id, UPPERCASE('open') AS status)\n" + "SELECT 1\n" +) + + +@pytest.mark.parametrize( + "test_case", + ( + ResourceSqlValidationCase( + description="singular audit unknown column and function", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + 'AUDIT ();\nSELECT o.order_id\nFROM __ref("orders") AS o\n' + 'JOIN __ref("customers") AS c ON o.order_id = c.order_id\n' + "WHERE o.shipped_at IS NULL AND STARTSWITH(o.status, 'x')\n", + ), + ), + expected_diagnostics=( + ( + "B002", + "audit 'orders_have_customers': " + "Unknown column 'shipped_at' in table 'orders' (context: WHERE)", + RESOURCE_SQL_SINGULAR_AUDIT, + 5, + ), + ( + "B101", + "audit 'orders_have_customers': " + "Unknown function 'STARTSWITH' for dialect DuckDB; did you mean STARTS_WITH?", + RESOURCE_SQL_SINGULAR_AUDIT, + 5, + ), + ), + ), + ResourceSqlValidationCase( + description="generic audit instance unknown column", + files=( + resource_sql_orders_model(", audits [status_is_known]"), + ( + RESOURCE_SQL_GENERIC_AUDIT, + "AUDIT ();\nSELECT * FROM @relation AS r\nWHERE r.status_code = 'unknown'\n", + ), + ), + expected_diagnostics=( + ( + "B002", + "audit 'status_is_known' on model 'orders': " + "Unknown column 'status_code' in table 'orders' (context: WHERE)", + RESOURCE_SQL_GENERIC_AUDIT, + 3, + ), + ), + ), + ResourceSqlValidationCase( + description="SQL test fixture and expected CTE unknown functions", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (\n" + " SELECT 1 AS order_id, STRING_TO_INT('7') AS customer_id\n),\n" + "__expected__orders AS (SELECT 1 AS order_id, UPPERCASE('open') AS status)\n" + "SELECT 1\n", + ), + ), + expected_diagnostics=( + ( + "B101", + "SQL test 'test_orders': Unknown function 'STRING_TO_INT' for dialect DuckDB", + RESOURCE_SQL_TEST, + 3, + ), + ( + "B101", + "SQL test 'test_orders': Unknown function 'UPPERCASE' for dialect DuckDB", + RESOURCE_SQL_TEST, + 5, + ), + ), + ), + ResourceSqlValidationCase( + description="SQL test assertion unknown model column", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (SELECT 1 AS order_id, 7 AS customer_id),\n" + '__assert__orders_are_open AS (\n SELECT * FROM __ref("orders") AS o\n' + " WHERE o.order_status <> 'open'\n)\nSELECT 1\n", + ), + ), + expected_diagnostics=( + ( + "B002", + "SQL test 'test_orders': " + "Unknown column 'order_status' in table 'orders' (context: WHERE)", + RESOURCE_SQL_TEST, + 5, + ), + ), + ), + ResourceSqlValidationCase( + description="inline and named SQL hook unknown functions", + files=( + resource_sql_orders_model( + ', post_hooks [inline_sql("SELECT STRING_TO_INT(\'1\')"), sql("record_orders")]' + ), + ( + RESOURCE_SQL_NAMED_HOOK, + 'HOOK (description "Count orders");\n' + 'SELECT COUNT_ROWS(*) FROM __ref("customers")\n', + ), + ), + expected_diagnostics=( + ( + "B101", + "post_hooks[0] SQL hook: Unknown function 'STRING_TO_INT' for dialect DuckDB", + RESOURCE_SQL_ORDERS, + 1, + ), + ( + "B101", + "post_hooks[1] SQL hook: Unknown function 'COUNT_ROWS' for dialect DuckDB", + RESOURCE_SQL_NAMED_HOOK, + 2, + ), + ), + ), + ResourceSqlValidationCase( + description="unparseable singular audit is a syntax error", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + 'AUDIT ();\nSELECT o.order_id\nFROM __ref("orders") AS o\n' + 'JOIN __ref("customers") AS c ON o.order_id = c.order_id\n' + "WHERE o.status ===== 'open'\n", + ), + ), + expected_diagnostics=( + ( + "P001", + "SQL syntax error in audit 'orders_have_customers': Unexpected token: Eq", + RESOURCE_SQL_SINGULAR_AUDIT, + 5, + ), + ), + ), + ResourceSqlValidationCase( + description="built-in column audit on a column the model does not output", + files=(resource_sql_orders_model(", columns (shipped_at (audits [not_null]))"),), + expected_diagnostics=( + ( + "B002", + "audit 'not_null' on model 'orders' column 'shipped_at': " + "Unknown column 'shipped_at' in table 'orders' (context: SELECT)", + "audits/generic/not_null.sql", + 3, + ), + ( + "B002", + "audit 'not_null' on model 'orders' column 'shipped_at': " + "Unknown column 'shipped_at' in table 'orders' (context: SELECT)", + "audits/generic/not_null.sql", + 5, + ), + ( + "K001", + "declared column 'shipped_at' was not found in statically inferred output " + "for model 'orders'", + RESOURCE_SQL_ORDERS, + 1, + ), + ), + ), + ResourceSqlValidationCase( + description="unparseable SQL test fixture is a syntax error", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (\n" + " SELECT 1 AS order_id, 7 AS customer_id FROM WHERE\n),\n" + "__expected__orders AS (SELECT 1 AS order_id)\nSELECT 1\n", + ), + ), + expected_diagnostics=( + ( + "P001", + "SQL syntax error in SQL test 'test_orders': " + "Expected table name or subquery, got Where", + RESOURCE_SQL_TEST, + 3, + ), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_resource_sql_error_when_compiling_and_building_then_command_fails_at_the_resource( + test_case: ResourceSqlValidationCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files=resource_sql_project_files(test_case.files), + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + built: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "build") + ) + + payload: dict[str, Any] = json.loads(compiled.stdout) + assert compiled.returncode == 1, compiled.stdout + compiled.stderr + assert ( + tuple( + (item["code"], item["message"], item["path"], item["line"]) + for item in payload["diagnostics"] + ) + == test_case.expected_diagnostics + ) + assert built.returncode == 1, built.stdout + built.stderr + assert not (project_dir / "warehouse.duckdb").exists() + + +@pytest.mark.parametrize( + "test_case", + ( + ResourceSqlValidationCase( + description="valid audits, SQL tests and hooks", + files=( + resource_sql_orders_model( + ", audits [status_is_known], " + "post_hooks [inline_sql(\"SELECT STARTS_WITH('open', 'o')\")]" + ), + ( + RESOURCE_SQL_GENERIC_AUDIT, + "AUDIT ();\nSELECT * FROM @relation AS r\nWHERE r.status <> 'open'\n", + ), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + 'AUDIT ();\nSELECT o.order_id\nFROM __ref("orders") AS o\n' + 'LEFT JOIN __ref("customers") AS c ON o.order_id = c.order_id\n' + "WHERE c.customer_id IS NULL\n", + ), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (SELECT 1 AS order_id, 7 AS customer_id),\n" + "__expected__orders AS (SELECT 1 AS order_id, LOWER('OPEN') AS status)\n" + "SELECT 1\n", + ), + ), + expected_diagnostics=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_valid_audits_tests_and_hooks_when_building_then_checks_pass( + test_case: ResourceSqlValidationCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files=resource_sql_project_files(test_case.files), + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + built: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "build") + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert tuple(json.loads(compiled.stdout)["diagnostics"]) == test_case.expected_diagnostics + assert built.returncode == 0, built.stdout + built.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + ResourceSqlHelpCase( + description="generic audit on a model compares a text argument with a date", + files=( + ( + RESOURCE_SQL_ORDERS, + 'MODEL (description "Orders", audits [recent_rows (date_column "ordered_on")]);\n' + "SELECT CAST(c.order_id AS INTEGER) AS order_id," + " CAST('2026-04-01' AS VARCHAR) AS ordered_on\n" + 'FROM __ref("customers") AS c\n', + ), + ( + RESOURCE_SQL_RECENT_ROWS_AUDIT, + "AUDIT ();\nSELECT 1 AS missing_rows FROM @relation AS r\n" + "WHERE @date_column = CURRENT_DATE\n", + ), + ), + expected_diagnostics=( + ( + "B217", + "audit 'recent_rows' on model 'orders': " + "Incompatible comparison between STRING and DATE", + "Convert the text value explicitly, for example `CAST(@date_column AS DATE)`, " + "or attach audit 'recent_rows' to a DATE column", + ), + ), + ), + ResourceSqlHelpCase( + description="SQL test reads a column the model no longer outputs", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (SELECT 1 AS order_id, 7 AS customer_id),\n" + '__assert__orders_are_open AS (\n SELECT * FROM __ref("orders") AS o\n' + " WHERE o.order_status <> 'open'\n)\nSELECT 1\n", + ), + ), + expected_diagnostics=( + ( + "B002", + "SQL test 'test_orders': " + "Unknown column 'order_status' in table 'orders' (context: WHERE)", + "'orders' does not output 'order_status' (it outputs order_id, status); " + "update the test to its current columns", + ), + ), + ), + ResourceSqlHelpCase( + description="SQL test assertion compares text values with model columns", + files=( + ("sqlbuild_project.toml", 'name = "orders"\nadapter = "snowflake"\n'), + ( + f"{RESOURCE_SQL_MODELS}/customers.sql", + 'MODEL (description "Customers", database example, schema analytics);\n' + "SELECT CAST(1 AS INTEGER) AS order_id\n", + ), + ( + RESOURCE_SQL_ORDERS, + 'MODEL (description "Orders", database example, schema analytics);\n' + "SELECT CAST(c.order_id AS INTEGER) AS order_id," + " CAST('2026-04-01' AS TIMESTAMP) AS ordered_at\n" + 'FROM __ref("customers") AS c\n', + ), + ( + RESOURCE_SQL_TEST, + "TEST ();\nWITH __ref__customers AS (SELECT 1 AS order_id),\n" + "__assert__orders_match AS (\n" + ' SELECT o.order_id, o.ordered_at FROM __ref("orders") AS o\n' + " EXCEPT\n" + " SELECT CAST('1' AS VARCHAR) AS order_id, '2026-04-01' AS ordered_at\n" + ")\nSELECT 1\n", + ), + ), + expected_diagnostics=( + ( + "B215", + "SQL test 'test_orders': Set-operation column 1 may fail during runtime " + "conversion: accumulated type NUMBER, next type VARCHAR", + "Cast set-operation column 1 to the tested model's type INT in the " + "expected or fixture rows, for example `CAST('1' AS INT)`", + ), + ( + "B215", + "SQL test 'test_orders': Set-operation column 2 may fail during runtime " + "conversion: accumulated type TIMESTAMP_NTZ, next type VARCHAR", + "Cast set-operation column 2 to the tested model's type TIMESTAMP in the " + "expected or fixture rows, for example `CAST('2026-04-01' AS TIMESTAMP)`", + ), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_typed_resource_mismatch_when_compiling_then_message_names_owner_and_shows_cast( + test_case: ResourceSqlHelpCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files=resource_sql_project_files(test_case.files), + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + + payload: dict[str, Any] = json.loads(compiled.stdout) + assert compiled.returncode == 1, compiled.stdout + compiled.stderr + assert ( + tuple((item["code"], item["message"], item.get("help")) for item in payload["diagnostics"]) + == test_case.expected_diagnostics + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ResourceSqlOptOutCase( + description="--no-sql-analysis skips audit and SQL test checks", + files=( + resource_sql_orders_model(), + (RESOURCE_SQL_SINGULAR_AUDIT, "AUDIT ();\n" + _FINDING_AUDIT_BODY), + (RESOURCE_SQL_TEST, "TEST ();\n" + _FINDING_TEST_BODY), + ), + flags=("--no-sql-analysis",), + expected_diagnostics=(), + ), + ResourceSqlOptOutCase( + description="AUDIT and TEST header opt-outs skip their checks", + files=( + resource_sql_orders_model(), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + "AUDIT (sql_analysis false);\n" + _FINDING_AUDIT_BODY, + ), + (RESOURCE_SQL_TEST, "TEST (sql_analysis false);\n" + _FINDING_TEST_BODY), + ), + flags=(), + expected_diagnostics=(), + ), + ResourceSqlOptOutCase( + description="SQL tests of a model with sql_analysis false are not analysed", + files=( + resource_sql_orders_model(", sql_analysis false"), + (RESOURCE_SQL_TEST, "TEST ();\n" + _FINDING_TEST_BODY), + ), + flags=(), + expected_diagnostics=(), + ), + ResourceSqlOptOutCase( + description="required analysis rejects opt-outs on parseable audits and tests", + files=( + _REQUIRED_ANALYSIS_TOML, + resource_sql_orders_model(), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + "AUDIT (sql_analysis false);\n" + _FINDING_AUDIT_BODY, + ), + (RESOURCE_SQL_TEST, "TEST (sql_analysis false);\n" + _FINDING_TEST_BODY), + ), + flags=(), + expected_diagnostics=( + ("P009", RESOURCE_SQL_SINGULAR_AUDIT, 1), + ("P009", RESOURCE_SQL_TEST, 1), + ), + expected_returncode=1, + ), + ResourceSqlOptOutCase( + description="required analysis accepts an opt-out on an unparseable audit", + files=( + _REQUIRED_ANALYSIS_TOML, + resource_sql_orders_model(), + ( + RESOURCE_SQL_SINGULAR_AUDIT, + "AUDIT (sql_analysis false);\n" + + _FINDING_AUDIT_BODY.replace("STARTSWITH(o.status, 'x')", "o.status ===== 1"), + ), + ), + flags=(), + expected_diagnostics=(), + ), + ResourceSqlOptOutCase( + description="required analysis accepts a model opt-out for an unparseable hook", + files=( + _REQUIRED_ANALYSIS_TOML, + resource_sql_orders_model( + ', sql_analysis false, post_hooks [inline_sql("CHECKPOINT")]' + ), + ), + flags=(), + expected_diagnostics=(), + ), + ResourceSqlOptOutCase( + description="required analysis rejects a model opt-out when its hooks parse", + files=( + _REQUIRED_ANALYSIS_TOML, + resource_sql_orders_model( + ', sql_analysis false, post_hooks [inline_sql("SELECT 1")]' + ), + ), + flags=(), + expected_diagnostics=(("P009", RESOURCE_SQL_ORDERS, 1),), + expected_returncode=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_analysis_opt_outs_when_compiling_then_resource_checks_follow_them( + test_case: ResourceSqlOptOutCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files=resource_sql_project_files(test_case.files), + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, + command=("--no-color", "compile", "--json", "--no-cache", *test_case.flags), + ) + + payload: dict[str, Any] = json.loads(compiled.stdout) + assert ( + tuple((item["code"], item["path"], item["line"]) for item in payload["diagnostics"]) + == test_case.expected_diagnostics + ), compiled.stdout + compiled.stderr + assert compiled.returncode == test_case.expected_returncode + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_type_finding_locations.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_type_finding_locations.py new file mode 100644 index 0000000000..139daa794c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_type_finding_locations.py @@ -0,0 +1,90 @@ +"""E2E coverage of authored locations for cast and set-operation type findings.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import TypeFindingLocationCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + ( + TypeFindingLocationCase( + description="two failing literal casts report two findings at their casts", + adapter="duckdb", + query_sql=( + "WITH final AS (\n" + " SELECT\n" + " CAST(TIMESTAMP '2026-04-01' AS INTEGER) AS ordered_epoch,\n" + " CAST(TIMESTAMP '2026-04-02' AS INTEGER) AS shipped_epoch\n" + ")\n" + "SELECT f.ordered_epoch, f.shipped_epoch FROM final AS f\n" + ), + expected_diagnostics=( + ("B218", "Cannot cast TIMESTAMP to INTEGER", 5, 5), + ("B218", "Cannot cast TIMESTAMP to INTEGER", 6, 5), + ), + ), + TypeFindingLocationCase( + description="set-operation coercion points at the branch projection", + adapter="snowflake", + query_sql=( + "WITH final AS (\n" + " SELECT 1 AS order_id\n" + " UNION ALL\n" + " SELECT 'pending'\n" + ")\n" + "SELECT f.order_id FROM final AS f\n" + ), + expected_diagnostics=( + ( + "B215", + "Set-operation column 1 may fail during runtime conversion: " + "accumulated type NUMBER, next type VARCHAR", + 6, + 10, + ), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_type_findings_without_native_spans_when_compiling_then_each_is_located( + test_case: TypeFindingLocationCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": f'name = "orders"\nadapter = "{test_case.adapter}"\n', + "models/orders.sql": ( + 'MODEL (description "Order epochs", database warehouse, schema analytics);\n\n' + + test_case.query_sql + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "compile", "--json", "--no-cache") + ) + + payload: dict[str, Any] = json.loads(result.stdout) + assert result.returncode == 1, result.stdout + result.stderr + assert ( + tuple( + (item["code"], item["message"], item["line"], item["column"]) + for item in payload["diagnostics"] + ) + == test_case.expected_diagnostics + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan.py index 1453a1eab6..729c2b6b7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_plan.py @@ -208,7 +208,7 @@ def test_given_dbt_interop_project_when_running_human_plan_then_outputs_grouped_ command=("dbt", "plan", "--json", "--select", "anything"), expected_stderr_fragments=( "error[C240]: dbt project directory is not configured", - "= help: Add [dbt].project_dir", + '= help: to point SQLBuild at the dbt project, set this in sqlbuild_project.toml:\n [dbt]\n project_dir = "path/to/dbt_project"', ), ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 201fc414a1..1be089a3c1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -433,7 +433,10 @@ def test_given_direct_state_history_when_running_janitor_then_it_prunes_history( description="tracked-only janitor requires query tracking", command=("janitor", "--auto-approve"), expected_exit_code=1, - expected_stderr_fragments=("janitor.delete_tracked_only requires",), + expected_stderr_fragments=( + "[janitor] delete_tracked_only = true requires query change tracking", + " query_change_tracking = true", + ), ) ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 37139e5183..6a4bb69c45 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -36,6 +36,7 @@ class UnevaluatedResourceCase: expected_reason: str = "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED" adapter: str = "duckdb" extra_files: tuple[tuple[str, str], ...] = () + rule: str = "SQBRSQL035" @dataclass(frozen=True) @@ -138,3 +139,53 @@ class SqlRulePathParityCase: files: dict[str, str] expected_findings: tuple[tuple[str, int, int, str], ...] expected_exit: int = 1 + + +@dataclass(frozen=True) +class SqlQualityRuleCase: + description: str + upstream_header: str + query_sql: str + expected_findings: tuple[tuple[str, int], ...] + expected_after_fix: tuple[tuple[str, int], ...] + + +@dataclass(frozen=True) +class RankingKeyProofCase: + description: str + upstream_header: str + upstream_sql: str + query_sql: str + expected_codes: tuple[str, ...] + extra_files: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True) +class RankingSortDefaultCase: + description: str + order_by: str + expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class FixableReportingCase: + description: str + summary_sql: str + expected_fixability: tuple[tuple[str, bool], ...] + expected_notes: tuple[tuple[str, tuple[str, ...]], ...] + expected_help_fragments: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class ConstantLiteralCase: + description: str + summary_sql: str + expected_exit: int + expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class DuplicateLiteralCase: + description: str + literals: tuple[tuple[str, str], ...] + expected_hints: tuple[tuple[str, str | None], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 659ca60fa4..4f9e255e1e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -165,3 +165,64 @@ def custom_rule_messages(result: subprocess.CompletedProcess[str]) -> tuple[str, """Return custom-rule diagnostic messages from one JSON compile result.""" return tuple(str(item["message"]) for item in custom_rule_diagnostics(result)) + + +def write_quality_project(*, project_dir: Path, toml: str, upstream: str, summary: str) -> None: + """Write a two-model project: `raw_orders` and the `order_summary` model under test.""" + + (project_dir / "sqlbuild_project.toml").write_text(toml, encoding="utf-8") + models: Path = project_dir / "models" + models.mkdir() + (models / "raw_orders.sql").write_text(upstream, encoding="utf-8") + (models / "order_summary.sql").write_text(summary, encoding="utf-8") + + +def diagnostic_codes(result: subprocess.CompletedProcess[str]) -> tuple[str, ...]: + """Return diagnostic codes from one JSON compile result, in report order.""" + + return tuple(str(item["code"]) for item in json.loads(result.stdout)["diagnostics"]) + + +def finding_fixability(result: subprocess.CompletedProcess[str]) -> dict[str, bool]: + """Map each code in a `rules --json run` result to whether its finding reports a fix.""" + + return { + str(item["code"]): item.get("fixable") is True + for item in json.loads(result.stdout)["findings"] + } + + +def fixable_finding_count(result: subprocess.CompletedProcess[str]) -> int: + """Count findings that report an available fix in a `rules --json run` result.""" + + return sum(item.get("fixable") is True for item in json.loads(result.stdout)["findings"]) + + +def diagnostic_notes(result: subprocess.CompletedProcess[str]) -> dict[str, tuple[str, ...]]: + """Map each code in a JSON compile result to its diagnostic notes.""" + + return { + str(item["code"]): tuple(item.get("notes", ())) + for item in json.loads(result.stdout)["diagnostics"] + } + + +def help_fragment_presence( + result: subprocess.CompletedProcess[str], fragments: tuple[tuple[str, str], ...] +) -> tuple[bool, ...]: + """Return, per `(code, fragment)`, whether that code's diagnostic help contains it.""" + + helps: dict[str, str] = { + str(item["code"]): str(item.get("help", "")) + for item in json.loads(result.stdout)["diagnostics"] + } + return tuple(fragment in helps.get(code, "") for code, fragment in fragments) + + +def long_literal_hints(result: subprocess.CompletedProcess[str]) -> dict[str, str]: + """Map each finding path of a SQBRSQL044-only run to the help before the generic guidance.""" + + return { + str(item["path"]): str(item["remediation"]).partition("Define the value once")[0].strip() + for item in json.loads(result.stdout)["findings"] + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py new file mode 100644 index 0000000000..2efe6d27b9 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_sql_quality_rules.py @@ -0,0 +1,546 @@ +"""E2E coverage of literal length, ranking caps, unused CTE outputs and ranking proofs.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + ConstantLiteralCase, + DuplicateLiteralCase, + FixableReportingCase, + RankingKeyProofCase, + RankingSortDefaultCase, + SqlQualityRuleCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + diagnostic_codes, + diagnostic_notes, + finding_fixability, + fixable_finding_count, + help_fragment_presence, + long_literal_hints, + run_unevaluated_rules_cli, + write_quality_project, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\n' + 'select = ["SQBRSQL018", "SQBRSQL042", "SQBRSQL043", "SQBRSQL044"]\n\n' + "[rules.thresholds]\nmax_literal_length = 30\nmax_ranking_order_by = 2\n" +) +_UPSTREAM_SQL: str = ( + "SELECT 1 AS order_id, 7 AS customer_id, 'open' AS status, 'n' AS note, 3 AS amount\n" +) +_SUMMARY_HEADER: str = 'MODEL (description "Order summary");\n' +_SHARED_LITERAL: str = "'this order is still waiting for payment'" +_OTHER_LITERAL: str = "'this ticket is still waiting for a reply'" +_RAW_HEADER: str = 'MODEL (description "Raw orders");\n' +_KEYED_RAW_HEADER: str = ( + 'MODEL (description "Raw orders", columns (order_id (audits [unique, not_null])));\n' +) +_FIX_NOTE: str = "Fix available: `sqb format --fix` applies it after compiler verification" +_RENAMED_CHAIN_SQL: str = ( + "WITH orders AS (\n" + " SELECT r.order_id AS sale_id, r.customer_id, r.amount\n" + ' FROM __ref("raw_orders") AS r\n' + "),\n" + "staged AS (\n" + " SELECT o.sale_id, o.customer_id, o.amount FROM orders AS o\n" + "),\n" + "final AS (\n" + " SELECT s.sale_id," + " ROW_NUMBER() OVER (PARTITION BY s.customer_id ORDER BY s.amount DESC, s.sale_id) AS rn\n" + " FROM staged AS s\n" + ")\n" + "SELECT f.sale_id, f.rn FROM final AS f\n" +) +_INCREMENTAL_HEADER_TEMPLATE: str = ( + "MODEL (\n" + ' description "Raw orders",\n' + " materialized incremental,\n" + " incremental_strategy {strategy},\n" + " unique_key order_id,\n" + "{cursor}" + ");\n" +) +_CURSOR_CONFIG: str = ( + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' +) +_INCREMENTAL_SQL: str = ( + "SELECT 1 AS order_id, 7 AS customer_id, 3 AS amount," + " TIMESTAMP '2026-01-01 00:00:00' AS order_date\n" +) +_SNAPSHOT_HEADER: str = ( + "MODEL (\n" + ' description "Raw orders",\n' + " materialized snapshot,\n" + " unique_key [order_id],\n" + " snapshot_strategy timestamp,\n" + " updated_at order_date,\n" + ");\n" +) +_KEYED_SOURCE_YML: str = ( + "sources:\n" + " - name: landing_orders\n" + " schema: landing\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + " audits: [unique, not_null]\n" +) + + +@pytest.mark.parametrize( + "test_case", + ( + SqlQualityRuleCase( + description="every quality Rule reports and the unused-column fix applies", + upstream_header='MODEL (description "Raw orders", columns (order_id (audits [unique, not_null])));\n', + query_sql=( + "WITH orders AS (\n" + " SELECT r.order_id, r.customer_id, r.status, r.note, r.amount\n" + ' FROM __ref("raw_orders") AS r\n' + "),\n" + "final AS (\n" + " SELECT\n" + " o.order_id,\n" + " CASE WHEN o.status = 'open' THEN 'this order is still waiting for payment'" + " ELSE 'done' END AS label,\n" + " ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount) AS amount_rank,\n" + " ROW_NUMBER() OVER (PARTITION BY o.customer_id" + " ORDER BY o.amount, o.status, o.order_id) AS stable_rank\n" + " FROM orders AS o\n" + ")\n" + "SELECT f.order_id, f.label, f.amount_rank, f.stable_rank FROM final AS f\n" + ), + expected_findings=( + ("SQBRSQL042", 3), + ("SQBRSQL044", 9), + ("SQBRSQL018", 10), + ("SQBRSQL043", 11), + ), + expected_after_fix=(("SQBRSQL044", 16), ("SQBRSQL018", 19), ("SQBRSQL043", 20)), + ), + SqlQualityRuleCase( + description="a ranking order without a declared key cannot be proven", + upstream_header='MODEL (description "Raw orders");\n', + query_sql=( + "WITH orders AS (\n" + " SELECT r.order_id, r.customer_id\n" + ' FROM __ref("raw_orders") AS r\n' + "),\n" + "final AS (\n" + " SELECT o.order_id," + " ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.order_id) AS rn\n" + " FROM orders AS o\n" + ")\n" + "SELECT f.order_id, f.rn FROM final AS f\n" + ), + expected_findings=(("SQBRSQL018", 7),), + expected_after_fix=(("SQBRSQL018", 12),), + ), + ), + ids=lambda case: case.description, +) +def test_given_quality_rule_violations_when_compiling_and_fixing_then_findings_and_fixes_match( + test_case: SqlQualityRuleCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") + models: Path = tmp_path / "models" + models.mkdir() + (models / "raw_orders.sql").write_text(test_case.upstream_header + _UPSTREAM_SQL) + (models / "order_summary.sql").write_text(_SUMMARY_HEADER + test_case.query_sql) + + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + fixed: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli(tmp_path, "format", "--fix") + recompiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + + before: list[dict[str, Any]] = json.loads(compiled.stdout)["diagnostics"] + after: list[dict[str, Any]] = json.loads(recompiled.stdout)["diagnostics"] + assert compiled.returncode == 1, compiled.stdout + compiled.stderr + assert tuple((item["code"], item["line"]) for item in before) == test_case.expected_findings + assert fixed.returncode == 0, fixed.stdout + fixed.stderr + assert tuple((item["code"], item["line"]) for item in after) == test_case.expected_after_fix + + +@pytest.mark.parametrize( + "test_case", + ( + RankingKeyProofCase( + description="a unique column audit proves the order through a rename and a CTE chain", + upstream_header=( + 'MODEL (description "Raw orders", columns (order_id (audits [unique, not_null])));\n' + ), + upstream_sql=_UPSTREAM_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=(), + ), + RankingKeyProofCase( + description="a merge unique_key proves the order through a rename", + upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( + strategy="merge", cursor=_CURSOR_CONFIG + ), + upstream_sql=_INCREMENTAL_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=(), + ), + RankingKeyProofCase( + description="a key-matched delete_insert unique_key proves the order", + upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( + strategy="delete_insert", cursor="" + ), + upstream_sql=_INCREMENTAL_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=(), + ), + RankingKeyProofCase( + description="a cursor-ranged delete_insert unique_key is not a key", + upstream_header=_INCREMENTAL_HEADER_TEMPLATE.format( + strategy="delete_insert", cursor=_CURSOR_CONFIG + ), + upstream_sql=_INCREMENTAL_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=("SQBRSQL018",), + ), + RankingKeyProofCase( + description="a snapshot unique_key identifies entities, not versions", + upstream_header=_SNAPSHOT_HEADER, + upstream_sql=_INCREMENTAL_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=("SQBRSQL018",), + ), + RankingKeyProofCase( + description="a unique audit on a nullable column is not a key", + upstream_header='MODEL (description "Raw orders", columns (order_id (audits [unique])));\n', + upstream_sql=_UPSTREAM_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=("SQBRSQL018",), + ), + RankingKeyProofCase( + description="a unique audit on a nullable false column is a key", + upstream_header=( + 'MODEL (description "Raw orders",' + " columns (order_id (nullable false, audits [unique])));\n" + ), + upstream_sql=_UPSTREAM_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=(), + ), + RankingKeyProofCase( + description="a declared source key proves the order of a source read", + upstream_header=_RAW_HEADER, + upstream_sql=_UPSTREAM_SQL, + query_sql=_RENAMED_CHAIN_SQL.replace( + '__ref("raw_orders")', '__source("landing_orders")' + ), + expected_codes=(), + extra_files=(("sources/landing.yml", _KEYED_SOURCE_YML),), + ), + RankingKeyProofCase( + description="a QUALIFY window cannot prove its own order", + upstream_header=_KEYED_RAW_HEADER, + upstream_sql=_UPSTREAM_SQL, + query_sql=( + "SELECT r.customer_id, r.order_id\n" + 'FROM __ref("raw_orders") AS r\n' + "QUALIFY ROW_NUMBER() OVER (PARTITION BY r.customer_id ORDER BY r.amount DESC) = 1\n" + ), + expected_codes=("SQBRSQL018",), + ), + RankingKeyProofCase( + description="an upstream hashed row key with a unique audit is a tie-breaker", + upstream_header=( + 'MODEL (description "Raw orders", columns (row_key (audits [unique, not_null])));\n' + ), + upstream_sql=("SELECT HASH(7, 11, 1) AS row_key, 7 AS customer_id, 3 AS amount\n"), + query_sql=( + "WITH orders AS (\n" + " SELECT r.row_key, r.customer_id, r.amount\n" + ' FROM __ref("raw_orders") AS r\n' + "),\n" + "final AS (\n" + " SELECT o.row_key," + " ROW_NUMBER() OVER (PARTITION BY o.customer_id ORDER BY o.amount, o.row_key) AS rn\n" + " FROM orders AS o\n" + ")\n" + "SELECT f.row_key, f.rn FROM final AS f\n" + ), + expected_codes=(), + ), + RankingKeyProofCase( + description="the renamed chain is unproven without a declared key", + upstream_header='MODEL (description "Raw orders");\n', + upstream_sql=_UPSTREAM_SQL, + query_sql=_RENAMED_CHAIN_SQL, + expected_codes=("SQBRSQL018",), + ), + ), + ids=lambda case: case.description, +) +def test_given_declared_upstream_keys_when_compiling_then_ranking_proof_follows_them( + test_case: RankingKeyProofCase, tmp_path: Path +) -> None: + write_quality_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + upstream=test_case.upstream_header + test_case.upstream_sql, + summary=_SUMMARY_HEADER + test_case.query_sql, + ) + for path, contents in test_case.extra_files: + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(contents, encoding="utf-8") + + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + + assert diagnostic_codes(compiled) == test_case.expected_codes, compiled.stdout + compiled.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + RankingSortDefaultCase( + description="five sort keys fit the default", + order_by="o.amount, o.status, o.note, o.customer_id, o.order_id", + expected_codes=(), + ), + RankingSortDefaultCase( + description="six sort keys exceed the default", + order_by="o.amount, o.status, o.note, o.customer_id, o.amount + 1, o.order_id", + expected_codes=("SQBRSQL043",), + ), + ), + ids=lambda case: case.description, +) +def test_given_default_ranking_limit_when_compiling_then_five_sort_keys_are_allowed( + test_case: RankingSortDefaultCase, tmp_path: Path +) -> None: + write_quality_project( + project_dir=tmp_path, + toml='name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL043"]\n', + upstream=_RAW_HEADER + _UPSTREAM_SQL, + summary=( + _SUMMARY_HEADER + "SELECT o.order_id," + f" ROW_NUMBER() OVER (ORDER BY {test_case.order_by}) AS rn\n" + 'FROM __ref("raw_orders") AS o\n' + ), + ) + + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + + assert diagnostic_codes(compiled) == test_case.expected_codes, compiled.stdout + compiled.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + FixableReportingCase( + description="only the unused-column finding reports a fix", + summary_sql=( + _SUMMARY_HEADER + "WITH orders AS (\n" + " SELECT r.order_id, r.note\n" + ' FROM __ref("raw_orders") AS r\n' + "),\n" + "final AS (\n" + " SELECT o.order_id, 'this order is still waiting for payment' AS label," + " ROW_NUMBER() OVER (ORDER BY o.order_id, o.order_id + 1, o.order_id + 2) AS rn\n" + " FROM orders AS o\n" + ")\n" + "SELECT f.order_id, f.label, f.rn FROM final AS f\n" + ), + expected_fixability=( + ("SQBRSQL042", True), + ("SQBRSQL043", False), + ("SQBRSQL044", False), + ), + expected_notes=( + ("SQBRSQL042", (_FIX_NOTE,)), + ("SQBRSQL043", ()), + ("SQBRSQL044", ()), + ), + expected_help_fragments=( + ("SQBRSQL043", "(the current value is 2)"), + ("SQBRSQL043", "[rules.thresholds]\n max_ranking_order_by = 3"), + ("SQBRSQL044", '@const("_status_pattern")'), + ("SQBRSQL044", "[rules.thresholds]\n max_literal_length = 39"), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_fixable_and_manual_findings_when_reporting_then_only_fixable_ones_say_so( + test_case: FixableReportingCase, tmp_path: Path +) -> None: + write_quality_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + upstream=_KEYED_RAW_HEADER + _UPSTREAM_SQL, + summary=test_case.summary_sql, + ) + + rules_json: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", "SQBRSQL" + ) + rules_text: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "run", "SQBRSQL" + ) + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + + fixability: dict[str, bool] = finding_fixability(rules_json) + assert tuple((code, fixability.get(code)) for code, _ in test_case.expected_fixability) == ( + test_case.expected_fixability + ) + assert rules_text.stdout.count(_FIX_NOTE) == fixable_finding_count(rules_json) + assert diagnostic_notes(compiled) == dict(test_case.expected_notes) + assert help_fragment_presence(compiled, test_case.expected_help_fragments) == (True,) * len( + test_case.expected_help_fragments + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ConstantLiteralCase( + description="a model-private constant satisfies the literal limit", + summary_sql=( + "MODEL (\n" + ' description "Order summary",\n' + ' constants (_waiting_label "this order is still waiting for payment"),\n' + ");\n" + 'SELECT o.order_id, @const("_waiting_label") AS label' + ' FROM __ref("raw_orders") AS o\n' + ), + expected_exit=0, + expected_codes=(), + ), + ConstantLiteralCase( + description="the same value inline exceeds the literal limit", + summary_sql=( + _SUMMARY_HEADER + + "SELECT o.order_id, 'this order is still waiting for payment' AS label" + ' FROM __ref("raw_orders") AS o\n' + ), + expected_exit=1, + expected_codes=("SQBRSQL044",), + ), + ), + ids=lambda case: case.description, +) +def test_given_long_value_as_constant_or_literal_when_compiling_then_only_literal_is_reported( + test_case: ConstantLiteralCase, tmp_path: Path +) -> None: + write_quality_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + upstream=_RAW_HEADER + _UPSTREAM_SQL, + summary=test_case.summary_sql, + ) + + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--json", "--no-cache" + ) + + assert compiled.returncode == test_case.expected_exit, compiled.stdout + compiled.stderr + assert diagnostic_codes(compiled) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + ( + DuplicateLiteralCase( + description="two models in one folder share a literal", + literals=( + ("models/sales/orders.sql", _SHARED_LITERAL), + ("models/sales/refunds.sql", _SHARED_LITERAL), + ("models/support/tickets.sql", _OTHER_LITERAL), + ), + expected_hints=( + ( + "models/sales/orders.sql", + "The identical literal also appears in models/sales/refunds.sql; their " + "nearest common folder is `models/sales`, so declare it once in " + "`models/sales/_sqlbuild/_constants/` and reference it with `@const` in " + "each file.", + ), + ( + "models/sales/refunds.sql", + "The identical literal also appears in models/sales/orders.sql; their " + "nearest common folder is `models/sales`, so declare it once in " + "`models/sales/_sqlbuild/_constants/` and reference it with `@const` in " + "each file.", + ), + ("models/support/tickets.sql", None), + ), + ), + DuplicateLiteralCase( + description="a subfolder use moves the declaration to the inherited role", + literals=( + ("models/sales/orders.sql", _SHARED_LITERAL), + ("models/sales/east/refunds.sql", _SHARED_LITERAL), + ), + expected_hints=( + ( + "models/sales/orders.sql", + "The identical literal also appears in models/sales/east/refunds.sql; their " + "nearest common folder is `models/sales`, so declare it once in " + "`models/sales/_sqlbuild/constants/` and reference it with `@const` in " + "each file.", + ), + ), + ), + DuplicateLiteralCase( + description="uses in different top folders need a project-wide constant", + literals=( + ("models/sales/orders.sql", _SHARED_LITERAL), + ("models/support/tickets.sql", _SHARED_LITERAL), + ), + expected_hints=( + ( + "models/sales/orders.sql", + "The identical literal also appears in models/support/tickets.sql; no " + "folder below the project roots holds every use, so declare it once in " + "top-level `constants/` and reference it with `@const` in each file.", + ), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_repeated_long_literal_when_running_rules_then_help_names_the_shared_location( + test_case: DuplicateLiteralCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") + for path, literal in test_case.literals: + model: Path = tmp_path / path + model.parent.mkdir(parents=True, exist_ok=True) + model.write_text(f"{_SUMMARY_HEADER}SELECT {literal} AS label\n", encoding="utf-8") + + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", "SQBRSQL044" + ) + + hints: dict[str, str] = long_literal_hints(result) + assert tuple((path, hints.get(path) or None) for path, _ in test_case.expected_hints) == ( + test_case.expected_hints + ), result.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index f1c2e49516..2d169f4577 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -36,6 +36,16 @@ expected_evaluated_models=2, extra_files=(("models/staging/customers.sql", "MODEL ();\nSELECT 1 AS order_id\n"),), ), + UnevaluatedResourceCase( + "model the compiler accepts but the Rules parser rejects", + "models/staging/orders.sql", + "MODEL ();\nWITH items AS (SELECT [1, 2] AS quantities)\n" + "SELECT 1 AS order_id, list_transform(i.quantities, lambda q: q + 1) AS shipped\n" + "FROM items AS i", + expected_evaluated_models=0, + expected_reason="could not parse models/staging/orders.sql for rules", + rule="SQBRMODEL101", + ), UnevaluatedResourceCase( "SQL-test guard", "tests/unit/test_orders.sql", @@ -54,10 +64,11 @@ def test_given_guarded_resource_when_running_rules_then_reports_failure_and_cove resource_template=test_case.template, adapter=test_case.adapter, extra_files=test_case.extra_files, + selected_rules=(test_case.rule,), ) for _ in range(2): result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "--json", "run", "SQBRSQL035" + tmp_path, "rules", "--json", "run", test_case.rule ) assert result.returncode == 1, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) @@ -66,10 +77,10 @@ def test_given_guarded_resource_when_running_rules_then_reports_failure_and_cove (finding,) = payload["findings"] assert finding["code"] == "rules-unevaluated" assert finding["path"] == test_case.path - assert finding["affected_rules"] == ["SQBRSQL035"] + assert finding["affected_rules"] == [test_case.rule] assert test_case.expected_reason in finding["message"] human: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( - tmp_path, "rules", "run", "SQBRSQL035" + tmp_path, "rules", "run", test_case.rule ) assert human.returncode == 1 assert ( @@ -190,7 +201,7 @@ def test_given_explicit_escape_hatch_when_running_rules_then_honors_policy( "Rules CLI groups selected family", ("rules", "--json", "run", "SQBRSQL"), "findings", - tuple(f"SQBRSQL{number:03}" for number in range(1, 42)), + tuple(f"SQBRSQL{number:03}" for number in range(1, 45)), ), GroupedRulesCase( "suppression precedes grouping", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py index f397a24b30..f22a73afe6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py @@ -32,9 +32,12 @@ ), expected_exit_code=1, expected_stderr_fragments=( - "error[C455]: scenario capture requires SQL analysis", - "= help: Enable settings.sql_analysis when capturing snapshots for local " - "scenario replay.", + "error[C455]: scenario capture requires SQL analysis; " + "sqlbuild_project.toml sets [settings] sql_analysis = false", + "= help: to capture snapshots for local scenario replay, " + "set this in sqlbuild_project.toml:\n" + " [settings]\n" + " sql_analysis = true", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index 042beb31ae..e1234511e1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -1727,9 +1727,12 @@ def test_given_unknown_scenario_selector_when_running_scenario_test_then_fails_c ), expected_exit_code=1, expected_stderr_fragments=( - "error[C455]: scenario test --local requires SQL analysis", - "= help: Enable settings.sql_analysis when running local scenario replay, " - "snapshot sync, or snapshot refresh.", + "error[C455]: scenario test --local requires SQL analysis; " + "sqlbuild_project.toml sets [settings] sql_analysis = false", + "= help: to run local scenario replay, snapshot sync, or snapshot refresh, " + "set this in sqlbuild_project.toml:\n" + " [settings]\n" + " sql_analysis = true", ), disabled_setting="sql_analysis", ), @@ -1745,9 +1748,12 @@ def test_given_unknown_scenario_selector_when_running_scenario_test_then_fails_c ), expected_exit_code=1, expected_stderr_fragments=( - "error[C455]: scenario test --local requires SQL analysis", - "= help: Enable settings.sql_analysis when running local scenario replay, " - "snapshot sync, or snapshot refresh.", + "error[C455]: scenario test --local requires SQL analysis; " + "sqlbuild_project.toml sets [settings] sql_analysis = false", + "= help: to run local scenario replay, snapshot sync, or snapshot refresh, " + "set this in sqlbuild_project.toml:\n" + " [settings]\n" + " sql_analysis = true", ), disabled_setting="sql_validation", ), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index c5d69931d5..5abecd9b99 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -853,3 +853,13 @@ class NativeAllocatorEntryTestCase: description: str inherited_environment: tuple[tuple[str, str], ...] expected_purge_delay: str + + +@dataclass(frozen=True) +class UnusedOutputDifferentialCase: + """One SQBRSQL042 CTE whose rows must survive `format --fix` unchanged.""" + + description: str + cte_sql: str + reader_sql: str + expected_statuses: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 82985c1d60..1c7ef28b7b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -256,6 +256,27 @@ def run_build( return exit_code, "".join(capsys.readouterr()) +def built_model_rows( + *, project_dir: Path, model: str, capsys: pytest.CaptureFixture[str] +) -> list[tuple[Any, ...]]: + """Build the project into its `orders.duckdb` and return one model's rows in a stable order.""" + + exit_code, output = run_build(project_dir=project_dir, flags=(), capsys=capsys) + assert exit_code == 0, output + with duckdb.connect(str(project_dir / "orders.duckdb"), read_only=True) as connection: + return sorted(connection.execute(f"SELECT * FROM {model}").fetchall(), key=repr) + + +def rule_fix_statuses(*, payload: dict[str, Any], code: str, file_name: str) -> tuple[str, ...]: + """Statuses of one Rule's fixes in one file, from `format --fix --json` output.""" + + key: tuple[str, str] = (code, file_name) + statuses: dict[tuple[str, str], list[str]] = {key: []} + for fix in payload["rule_fixes"]: + statuses.setdefault((fix["code"], Path(fix["file"]).name), []).append(fix["status"]) + return tuple(statuses[key]) + + def write_dropped_relation_project( *, project_dir: Path, model_name: str, model_sql: str, settings_toml: str = "" ) -> Path: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py index 43f3c23360..d1f246becd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_fix.py @@ -13,6 +13,21 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( FormatCompileIntegrationTestCase, SemanticFixTestCase, + UnusedOutputDifferentialCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + built_model_rows, + rule_fix_statuses, +) + +_DIFFERENTIAL_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_DIFFERENTIAL_RAW_SQL: str = ( + 'MODEL (description "Raw orders");\n' + "SELECT * FROM (VALUES (1, 7, 'open', 3, [1, 2]), (2, 7, 'done', 5, [3])," + " (3, 8, 'done', 3, [4, 5, 6]), (4, 9, 'done', 2, [7])) AS v" + " (order_id, customer_id, status, amount, tags)\n" ) @@ -237,6 +252,176 @@ def test_given_one_unverifiable_file_when_fixing_then_only_that_file_keeps_its_o } assert ("customers.sql", test_case.expected_literal) in faults assert ("orders.sql", test_case.expected_literal) not in faults + listed: list[tuple[str, str, int, str]] = [ + (fix["file"], fix["code"], fix["line"], fix["reason"]) for fix in payload["rule_fixes"] + ] + assert len(listed) == len(set(listed)) + + +@pytest.mark.parametrize( + "test_case", + [FormatCompileIntegrationTestCase("existing errors elsewhere", "UNION DISTINCT")], + ids=lambda case: case.description, +) +def test_given_existing_errors_and_unexpandable_file_when_fixing_then_other_files_are_fixed( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (tmp_path / "macros").mkdir() + (tmp_path / "macros" / "relations.py").write_text( + "def orders_relation() -> str:\n return '__ref(\"orders\")'\n" + ) + fixable: Path = models / "orders.sql" + fixable.write_text( + 'MODEL (description "Orders");\nSELECT 1 AS order_id UNION SELECT 2 AS order_id\n' + ) + unexpandable: Path = models / "customers.sql" + unexpandable_sql: str = ( + 'MODEL (description "Customers");\n' + "SELECT o.order_id AS customer_id FROM @orders_relation() AS o\n" + ) + unexpandable.write_text(unexpandable_sql) + (models / "payments.sql").write_text( + "MODEL (description \"Payments\");\nSELECT 1 AS payment_id WHERE 'open' > 1\n" + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert test_case.expected_literal in fixable.read_text() + assert unexpandable.read_text() == unexpandable_sql + assert { + (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] + } >= {("orders.sql", "SQBRSQL008", "applied")} + assert { + ( + Path(violation["file"]).name, + violation["code"], + violation["severity"], + "could not be expanded" in violation["message"], + ) + for violation in payload["violations"] + } >= {("customers.sql", "format-fix-skipped", "warning", True)} + + +@pytest.mark.parametrize( + "test_case", + [ + FormatCompileIntegrationTestCase( + "only an unverifiable fix", "Output columns could not be inferred" + ) + ], + ids=lambda case: case.description, +) +def test_given_only_unverifiable_fixes_when_fixing_then_every_refusal_is_listed_with_its_reason( + test_case: FormatCompileIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders.sql").write_text( + 'MODEL (description "Orders");\nSELECT 1 AS order_id WHERE 1 = NULL\n' + ) + (models / "customers.sql").write_text( + 'MODEL (description "Customers", sql_analysis false);\n' + "WITH unused AS (SELECT 2 AS customer_id) SELECT 1 AS customer_id\n" + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert { + (Path(fix["file"]).name, fix["code"], fix["status"]) for fix in payload["rule_fixes"] + } >= {("orders.sql", "SQBRSQL001", "refused"), ("customers.sql", "SQBRSQL005", "refused")} + assert { + (Path(violation["file"]).name, violation["message"]) for violation in payload["violations"] + } >= {("customers.sql", test_case.expected_literal + ", so the fix cannot be verified")} + + +@pytest.mark.parametrize( + "test_case", + [ + UnusedOutputDifferentialCase( + description="a plain column is removed", + cte_sql='SELECT r.order_id, r.status FROM __ref("raw_orders") AS r', + reader_sql="SELECT g.order_id FROM g", + expected_statuses=("applied",), + ), + UnusedOutputDifferentialCase( + description="GROUP BY ALL keeps its grouping column", + cte_sql=( + "SELECT r.customer_id, r.status, SUM(r.amount) AS total" + ' FROM __ref("raw_orders") AS r GROUP BY ALL' + ), + reader_sql="SELECT g.customer_id, g.total FROM g", + expected_statuses=("refused",), + ), + UnusedOutputDifferentialCase( + description="an ungrouped aggregate keeps the select at one row", + cte_sql="SELECT 'all' AS label, COUNT(*) AS n FROM __ref(\"raw_orders\") AS r", + reader_sql="SELECT g.label FROM g", + expected_statuses=("refused",), + ), + UnusedOutputDifferentialCase( + description="a set-returning projection keeps its row expansion", + cte_sql='SELECT r.order_id, UNNEST(r.tags) AS tag FROM __ref("raw_orders") AS r', + reader_sql="SELECT g.order_id FROM g", + expected_statuses=("refused",), + ), + UnusedOutputDifferentialCase( + description="ORDER BY ALL with LIMIT keeps its sort columns", + cte_sql=( + 'SELECT r.order_id, r.amount FROM __ref("raw_orders") AS r ORDER BY ALL LIMIT 2' + ), + reader_sql="SELECT g.amount FROM g", + expected_statuses=("refused",), + ), + UnusedOutputDifferentialCase( + description="a COLUMNS pattern reads every column", + cte_sql='SELECT r.order_id, r.status FROM __ref("raw_orders") AS r', + reader_sql="SELECT MAX(COLUMNS('^s')) FROM g", + expected_statuses=(), + ), + UnusedOutputDifferentialCase( + description="COLUMNS(*) reads every column", + cte_sql='SELECT r.order_id, r.amount FROM __ref("raw_orders") AS r', + reader_sql="SELECT MAX(COLUMNS(*)) FROM g", + expected_statuses=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_unused_cte_output_when_fixing_then_model_rows_are_unchanged( + test_case: UnusedOutputDifferentialCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_DIFFERENTIAL_TOML) + models: Path = tmp_path / "models" + models.mkdir() + (models / "raw_orders.sql").write_text(_DIFFERENTIAL_RAW_SQL) + (models / "summary.sql").write_text( + 'MODEL (description "Summary");\n' + f"WITH g AS ({test_case.cte_sql}),\nfinal AS ({test_case.reader_sql})\n" + "SELECT * FROM final\n" + ) + + before: list[tuple[Any, ...]] = built_model_rows( + project_dir=tmp_path, model="summary", capsys=capsys + ) + assert main(["--project-dir", str(tmp_path), "format", "--fix", "--json"]) in {0, 1} + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + after: list[tuple[Any, ...]] = built_model_rows( + project_dir=tmp_path, model="summary", capsys=capsys + ) + + assert after == before + assert ( + rule_fix_statuses(payload=payload, code="SQBRSQL042", file_name="summary.sql") + == test_case.expected_statuses + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py index cc8fdd53d2..9641558072 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py @@ -187,10 +187,11 @@ def test_given_macro_before_invalid_aggregate_when_compiling_then_reports_author "B218", ), SemanticCompileCase( - "identical span-less conversions are reported once", + "identical conversions are each reported at their own literal", _UPSTREAM, "SELECT 1 = 'not-a-number' AS first_result, 2 = 'not-a-number' AS second_result", "B218", + expected_diagnostic_count=2, ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py index 674bbfb1e0..c28eab2166 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_build_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_build_execution.py @@ -1010,7 +1010,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( "MODEL (\n materialized table\n" - " pre_hooks [inline_sql('INVALID SQL STATEMENT')]\n);\n\n" + " pre_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 1 AS id" ), "models/orders.sql": ( @@ -1032,7 +1032,7 @@ def test_given_view_with_run_audits_false_when_executing_then_succeeds( "sqlbuild_project.toml": _PROJECT_YML, "models/stg_orders.sql": ( "MODEL (\n materialized table\n" - " post_hooks [inline_sql('INVALID SQL STATEMENT')]\n);\n\n" + " post_hooks [inline_sql('SELECT * FROM missing_hook_table')]\n);\n\n" "SELECT 88 AS id" ), "models/orders.sql": ( diff --git a/tests/integration/src/sqlbuild/lint/__init__.py b/tests/integration/src/sqlbuild/lint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/lint/main/__init__.py b/tests/integration/src/sqlbuild/lint/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/lint/main/_test_types.py b/tests/integration/src/sqlbuild/lint/main/_test_types.py new file mode 100644 index 0000000000..dbe2a2f98c --- /dev/null +++ b/tests/integration/src/sqlbuild/lint/main/_test_types.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class FixVerdictTestCase: + description: str + before_models: tuple[tuple[str, str], ...] + after_models: tuple[tuple[str, str], ...] + edited: tuple[str, ...] + expected_failing: tuple[tuple[str, str], ...] + expected_unattributed: bool diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py new file mode 100644 index 0000000000..de2fac8de3 --- /dev/null +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -0,0 +1,35 @@ +"""Compile small DuckDB projects for Rule-fix verification tests.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.project import compile_project + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + + +def compile_models(*, project_dir: Path, models: tuple[tuple[str, str], ...]) -> CompiledProject: + """Write and compile the models.""" + + (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") + (project_dir / "models").mkdir(exist_ok=True) + for name, sql in models: + (project_dir / "models" / name).write_text(sql, encoding="utf-8") + return compile_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + ) + + +def without_lineage(project: CompiledProject) -> CompiledProject: + """The same compilation with column lineage unavailable for every model.""" + + return replace( + project, + models=tuple(replace(model, fast_lineage_columns=None) for model in project.models), + ) diff --git a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py new file mode 100644 index 0000000000..f3cba35506 --- /dev/null +++ b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py @@ -0,0 +1,142 @@ +"""Real-compile coverage for attributing Rule-fix compile differences to edited files.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.lint.main.compile_facts import compile_facts +from sqlbuild.lint.main.fix_verdict import fix_verdict +from sqlbuild.lint.models import FixVerdict +from tests.integration.src.sqlbuild.lint.main._test_types import FixVerdictTestCase +from tests.integration.src.sqlbuild.lint.main.helpers import compile_models, without_lineage + +_ORDERS: str = 'MODEL (description "Orders");\nSELECT CAST(1 AS INTEGER) AS order_id\n' +_REFORMATTED_ORDERS: str = _ORDERS + "\n" +_BROKEN_PAYMENTS: str = ( + "MODEL (description \"Payments\");\nSELECT 1 AS payment_id WHERE 'open' > 1\n" +) +_PAYMENTS: str = 'MODEL (description "Payments");\nSELECT 1 AS payment_id\n' +_CUSTOMERS: str = ( + 'MODEL (description "Customers");\nSELECT o.order_id AS customer_id FROM __ref("orders") AS o\n' +) + + +@pytest.mark.parametrize( + "test_case", + ( + FixVerdictTestCase( + description="an edit that adds a diagnostic fails its own file", + before_models=(("orders.sql", _ORDERS), ("payments.sql", _PAYMENTS)), + after_models=(("orders.sql", _ORDERS), ("payments.sql", _BROKEN_PAYMENTS)), + edited=("payments.sql",), + expected_failing=(("payments.sql", "The fix would add compile diagnostics: B218"),), + expected_unattributed=False, + ), + FixVerdictTestCase( + description="an edit that removes a diagnostic verifies", + before_models=(("orders.sql", _ORDERS), ("payments.sql", _BROKEN_PAYMENTS)), + after_models=(("orders.sql", _ORDERS), ("payments.sql", _PAYMENTS)), + edited=("payments.sql",), + expected_failing=(), + expected_unattributed=False, + ), + FixVerdictTestCase( + description="an existing error in another file does not block a verified edit", + before_models=(("orders.sql", _ORDERS), ("payments.sql", _BROKEN_PAYMENTS)), + after_models=( + ("orders.sql", _REFORMATTED_ORDERS), + ("payments.sql", _BROKEN_PAYMENTS), + ), + edited=("orders.sql",), + expected_failing=(), + expected_unattributed=False, + ), + FixVerdictTestCase( + description="an edit that changes an output type fails its file", + before_models=(("orders.sql", _ORDERS), ("customers.sql", _CUSTOMERS)), + after_models=( + ( + "orders.sql", + 'MODEL (description "Orders");\nSELECT CAST(1 AS VARCHAR) AS order_id\n', + ), + ("customers.sql", _CUSTOMERS), + ), + edited=("orders.sql",), + expected_failing=( + ("orders.sql", "The fix would change output columns, types or nullability"), + ), + expected_unattributed=True, + ), + ), + ids=lambda case: case.description, +) +def test_given_baseline_and_fixed_compiles_when_comparing_then_failures_belong_to_edited_files( + test_case: FixVerdictTestCase, tmp_path: Path +) -> None: + before_dir: Path = tmp_path / "before" + after_dir: Path = tmp_path / "after" + before_dir.mkdir() + after_dir.mkdir() + before: CompiledProject = compile_models(project_dir=before_dir, models=test_case.before_models) + after: CompiledProject = compile_models(project_dir=after_dir, models=test_case.after_models) + + verdict: FixVerdict = fix_verdict( + before=compile_facts(project=before, project_dir=before_dir), + after=compile_facts(project=after, project_dir=before_dir), + edited=frozenset((before_dir / "models" / name).resolve() for name in test_case.edited), + ) + + assert {path.name: reason for path, reason in verdict.failing.items()} == dict( + test_case.expected_failing + ) + assert verdict.unattributed is test_case.expected_unattributed + + +@pytest.mark.parametrize( + "test_case", + ( + FixVerdictTestCase( + description="lineage absent on both sides counts as unchanged when columns match", + before_models=(("orders.sql", _ORDERS), ("customers.sql", _CUSTOMERS)), + after_models=( + ("orders.sql", _REFORMATTED_ORDERS), + ("customers.sql", _CUSTOMERS), + ), + edited=("orders.sql",), + expected_failing=(), + expected_unattributed=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_lineage_unavailable_on_both_sides_when_comparing_then_matching_columns_verify( + test_case: FixVerdictTestCase, tmp_path: Path +) -> None: + before_dir: Path = tmp_path / "before" + after_dir: Path = tmp_path / "after" + before_dir.mkdir() + after_dir.mkdir() + before: CompiledProject = without_lineage( + compile_models(project_dir=before_dir, models=test_case.before_models) + ) + after: CompiledProject = without_lineage( + compile_models(project_dir=after_dir, models=test_case.after_models) + ) + + verdict: FixVerdict = fix_verdict( + before=compile_facts(project=before, project_dir=before_dir), + after=compile_facts(project=after, project_dir=before_dir), + edited=frozenset((before_dir / "models" / name).resolve() for name in test_case.edited), + ) + + assert {path.name: reason for path, reason in verdict.failing.items()} == dict( + test_case.expected_failing + ) + assert verdict.unattributed is test_case.expected_unattributed + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index df192e6338..9fc55c0c80 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -43,7 +43,7 @@ expected_warning_fragment=( "Warning: DuckDB adapter is active, but connection contains " "Snowflake-like keys: role, warehouse. If this is a Snowflake local config, " - "add top-level `adapter: snowflake` to sqlbuild_local.toml." + 'add `adapter = "snowflake"` at the top of sqlbuild_local.toml, before any section.' ), ) ], diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index 56dbad3621..13e530baf3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -79,7 +79,7 @@ 'depends_on=model("customers")', "if 'customers' is an external table that shares the name, qualify it with its " "schema", - "[references] enforce_explicit = false", + "[references]\n enforce_explicit = false", ), ), PythonSqlReferenceErrorTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index f787e23df3..c1e5616c6a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -125,7 +125,7 @@ def test_given_typed_reference_arguments_when_expanding_then_renders_call_site_r "model:order_summary depends on model:stg_orders through macro orders_base()", "--> macros/common.py", 'pass it in: @orders_base(__ref("stg_orders"))', - "[references] enforce_explicit = false", + "[references]\n enforce_explicit = false", ), ), MacroGeneratedReferenceErrorTestCase( @@ -134,7 +134,7 @@ def test_given_typed_reference_arguments_when_expanding_then_renders_call_site_r sql="@wrap('__ref(\"stg_orders\")')", expected_error_fragments=( "model:order_summary depends on model:stg_orders through macro wrap()", - "[references] enforce_explicit = false", + "[references]\n enforce_explicit = false", ), ), MacroGeneratedReferenceErrorTestCase( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index d4e94e1f67..fca7e0c412 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -1653,7 +1653,7 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( [janitor] enabled = true """.strip(), - expected_error_fragment="janitor.delete_tracked_only requires", + expected_error_fragment="delete_tracked_only = true requires query change tracking", ), LoadProjectConfigErrorTestCase( description="raises when janitor direct state history versions is negative", diff --git a/tests/unit/src/sqlbuild/errors/__init__.py b/tests/unit/src/sqlbuild/errors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/errors/setting_help/__init__.py b/tests/unit/src/sqlbuild/errors/setting_help/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/__init__.py b/tests/unit/src/sqlbuild/errors/setting_help/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/_test_types.py b/tests/unit/src/sqlbuild/errors/setting_help/main/_test_types.py new file mode 100644 index 0000000000..915224e9d6 --- /dev/null +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/_test_types.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class SettingHelpTestCase: + description: str + rendered: str + expected_text: str + + +@dataclass(frozen=True) +class SettingSnippetTestCase: + description: str + help_text: str + expected_section: str + expected_key: str + expected_value: object + + +@dataclass(frozen=True) +class SettingCatalogueTestCase: + description: str + repository_root: Path + expected_findings: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py new file mode 100644 index 0000000000..6702a216b9 --- /dev/null +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.setting_help.main.find_settings_without_snippet import find_settings_without_snippet +from scripts.setting_help.main.parse_setting_snippet import parse_setting_snippet +from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help +from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help +from sqlbuild.compiler.discovery.main.microbatch_guidance import ( + microbatch_guidance, +) +from sqlbuild.errors.setting_help.main.join_helps import join_helps +from sqlbuild.errors.setting_help.main.model_header_help import model_header_help +from sqlbuild.errors.setting_help.main.setting_help import setting_help +from sqlbuild.errors.setting_help.main.setting_note import setting_note +from tests.unit.src.sqlbuild.errors.setting_help.main._test_types import ( + SettingCatalogueTestCase, + SettingHelpTestCase, + SettingSnippetTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + SettingHelpTestCase( + "a boolean setting renders file, section and key", + setting_help( + purpose="to allow it", + file_name="sqlbuild_project.toml", + section="settings", + key="require_sql_analysis", + value=False, + ), + "to allow it, set this in sqlbuild_project.toml:\n" + " [settings]\n" + " require_sql_analysis = false", + ), + SettingHelpTestCase( + "a string setting is quoted", + setting_help( + purpose="to stage", + file_name="sqlbuild_project.toml", + section="settings", + key="table_promotion_mode", + value="staged", + ), + "to stage, set this in sqlbuild_project.toml:\n [settings]\n" + ' table_promotion_mode = "staged"', + ), + SettingHelpTestCase( + "an explicit note states the current value", + setting_note( + file_name="sqlbuild_local.toml", + section="settings", + key="sql_analysis", + value=False, + ), + "sqlbuild_local.toml sets [settings] sql_analysis = false", + ), + SettingHelpTestCase( + "a default note says the value is the default", + setting_note( + file_name="sqlbuild_project.toml", + section="references", + key="enforce_explicit", + value=True, + explicit=False, + ), + "sqlbuild_project.toml does not set [references] enforce_explicit, " + "so it defaults to true", + ), + SettingHelpTestCase( + "a MODEL header help shows the exact entry", + model_header_help(purpose="to skip it", entry="sql_analysis false"), + "to skip it, add this to the MODEL header:\n MODEL (\n" + " sql_analysis false,\n ...\n );", + ), + SettingHelpTestCase( + "several helps keep one label each", + join_helps("first", "second"), + "first\n = help: second", + ), + ), + ids=lambda case: case.description, +) +def test_given_setting_inputs_when_rendering_help_then_shows_exact_text( + test_case: SettingHelpTestCase, +) -> None: + assert test_case.rendered == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + ( + SettingSnippetTestCase( + "microbatch concurrency", + microbatch_guidance()[1], + "settings", + "microbatch_concurrency", + True, + ), + SettingSnippetTestCase( + "explicit references", + explicit_references_help(allowed="macro-generated references"), + "references", + "enforce_explicit", + False, + ), + SettingSnippetTestCase( + "project-wide SQL analysis", + sql_analysis_opt_out_help(), + "settings", + "sql_analysis", + False, + ), + ), + ids=lambda case: case.description, +) +def test_given_setting_guidance_when_reading_snippet_then_it_is_valid_toml_for_the_setting( + test_case: SettingSnippetTestCase, +) -> None: + assert parse_setting_snippet(test_case.help_text) == ( + test_case.expected_section, + test_case.expected_key, + test_case.expected_value, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + SettingCatalogueTestCase( + "Python and Rust diagnostics in this repository", + Path(__file__).resolve().parents[7], + expected_findings=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_error_catalogue_when_scanning_help_text_then_every_named_setting_has_toml( + test_case: SettingCatalogueTestCase, +) -> None: + assert tuple(find_settings_without_snippet(test_case.repository_root)) == ( + test_case.expected_findings + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index aa2a8e87af..2a3d4432bd 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -155,7 +155,7 @@ def test_given_dbt_vars_sources_when_resolving_then_vars_are_merged( cli_project_dir=None, expected_error_fragment="dbt project directory is not configured", expected_code="C240", - expected_help_fragment="Add [dbt].project_dir", + expected_help_fragment='[dbt]\n project_dir = "path/to/dbt_project"', ) ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 8b63690055..7f6519a7dc 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -66,7 +66,7 @@ "SQBRTEST203", "SQBRTEST301", ) -SQL_CODES: tuple[str, ...] = tuple(f"SQBRSQL{number:03d}" for number in range(1, 42)) +SQL_CODES: tuple[str, ...] = tuple(f"SQBRSQL{number:03d}" for number in range(1, 45)) BUILT_IN_CODES: tuple[str, ...] = tuple(sorted((*RULES_BUILT_IN_CODES, *SQL_CODES))) STRUCTURE_CODES: tuple[str, ...] = ( "SQBRMODEL101", diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 4d1f5d9534..bafe69f9e4 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -100,12 +100,21 @@ that are known to preserve meaning, then formats the layout. `--fix --check` rep changes without writing files, and `--fix --diff` prints their diff. Edits are planned in memory and repeated until no more apply; overlapping edits wait for the next -pass. Every pass recompiles the project and compares output columns (including types and -nullability), dependencies, and column lineage of each edited model and everything downstream of it -against the original compilation. When a pass cannot be proved, the edits of each file are checked -on their own: files whose edits cannot be proved keep their original contents and report -`format-fix-verification-failed`, while the other files are still fixed. A cycle or the 128-pass -limit fails without writing any edit. +pass. Every pass recompiles the project and compares it with the original compilation, so a +project that already has compile errors can still be fixed: + +- An edited file may lose diagnostics but must not gain any. Diagnostics are compared by code, + resource and message, not by line. +- Each edited model must keep its output columns, types and nullability, dependencies, and column + lineage. Lineage that is unavailable both before and after counts as unchanged when the columns + match. +- No other file's diagnostics or model outputs may change. + +A failure is attributed to the edited file that caused it, and that file keeps its original +contents and reports `format-fix-verification-failed` with the reason, while the other files are +still fixed. When a difference cannot be attributed, each file's edits are checked on their own. A +file whose SQL cannot be expanded is skipped with a `format-fix-skipped` warning that gives the +reason, and the run continues. A cycle or the 128-pass limit fails without writing any edit. | Rule | Decision | Reason | | --- | --- | --- | @@ -120,6 +129,7 @@ limit fails without writing any edit. | SQBRSQL023 unused table alias | Refused | Not yet reviewed as meaning-preserving. | | SQBRSQL025 implicit inner join | Applied | A bare `JOIN ... ON` is an inner join. | | SQBRSQL030 boolean CASE simplification | Refused | The `COALESCE` rewrite is not proved portable to every supported dialect. | +| SQBRSQL042 unused CTE output | Applied when row-neutral | Only a column reference or deterministic scalar expression is removed, from a CTE without `DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an ungrouped aggregate, or positional `GROUP BY`/`ORDER BY`. Other removals could change the rows and are refused with the reason. | Edits that touch macro or intrinsic expansions are refused, and so are edits outside model SQL bodies, because recompiling cannot prove their output contract. Findings without a proposed edit diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 1453b9cfaa..93ff76ab90 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -408,6 +408,15 @@ The `4/4` indicates the audit passed for all 4 microbatch batches. If a model is not incremental, `delta_and_final` degrades to `final` automatically. +## Skipping SQL analysis + +Compile checks audit SQL like model SQL: unknown columns and functions, type mismatches and syntax +errors are compile errors. For SQL the parser cannot read, add `sql_analysis false` to the `AUDIT` +header of a singular or generic audit; the audit still runs. Audits attached to a model with +`sql_analysis false`, and every audit in a `--no-sql-analysis` run, are not analysed. With +`[settings] require_sql_analysis = true`, the opt-out is accepted only on audit SQL that cannot be +parsed (see [Requiring SQL analysis](/docs/concepts/models/configuration#requiring-sql-analysis)). + ## Running audits standalone ```bash diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 4e4e98360c..51b24356ae 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -52,6 +52,45 @@ A model cannot re-enable analysis when either broader gate disables it. The olde and `--no-sql-validation` spellings remain compatibility aliases; use `sql_analysis` for new configuration. +### Requiring SQL analysis + +A project can stop opt-outs from hiding real findings: + +```toml +# sqlbuild_project.toml +[settings] +require_sql_analysis = true # default false +``` + +With this setting, `sql_analysis false` from a MODEL header or a `[path_defaults]` entry is accepted +only on a model whose SQL the parser cannot read: its query or one of its SQL hooks. On a model that +parses, compile reports a `P009` +error at the header key or `[path_defaults]` entry, with the findings the opt-out was hiding counted +by kind: + +```text +error[P009]: `sql_analysis false` is not needed for model 'orders_summary' + --> models/marts/orders_summary.sql:4:3 + = note: sqlbuild_project.toml sets [settings] require_sql_analysis = true, which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; this model parses successfully. + = help: remove `sql_analysis false` and fix the findings it was hiding: 2 unknown columns, 1 type mismatch (run `sqb compile` to see them) + = help: to allow `sql_analysis false` on any model, SQL test or audit, set this in sqlbuild_project.toml: + [settings] + require_sql_analysis = false +``` + +`TEST(...)` and `AUDIT(...)` headers (singular and generic audits) also accept +`sql_analysis false`, which skips compile-time SQL checks on that test or audit; the SQL still runs. +SQL tests of a model with `sql_analysis false`, and audits attached to it, are not analysed either. +`require_sql_analysis = true` applies the same rule to these headers: the opt-out is accepted only +when that test or audit SQL cannot be parsed, otherwise compile reports `P009` at the header key. + +`[settings] sql_analysis = false` in `sqlbuild_local.toml` is then a `D001` error, and +`require_sql_analysis` itself can only be set in `sqlbuild_project.toml`. `--no-sql-analysis` still +turns analysis off for one run, including the checks on audits, SQL tests and SQL hooks. + +When the parser cannot read a model, the `P001` syntax error shows the exact MODEL header entry +(`sql_analysis false,`) and asks you to report the SQL so the parser can support it. + ## Table fields | Field | Description | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 70642b7078..8a888be1b1 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -390,6 +390,7 @@ Global feature toggles: ```toml [settings] sql_analysis = true +require_sql_analysis = false query_change_tracking = true column_contract_mode = "implicit" concurrency = 1 @@ -403,6 +404,7 @@ default_audit_run_scope = "final" | Field | Default | Description | |-------|---------|-------------| | `sql_analysis` | `true` | Enable SQL syntax analysis, column binding, type inference, output inference, and semantic validation at compile time | +| `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | | `concurrency` | `1` | Maximum parallel model execution (currently serial only) | diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 36f6b8ea1c..44e414a286 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -103,6 +103,71 @@ owns the terminal `SELECT`'s plain projection from that last CTE, so select both the `SQBRSQL` family, for the complete convention. Parentheses around the entire query do not create a nested CTE scope. +## SQL quality limits and ranking determinism + +These Rules are errors with project-wide limits. No model can opt out of a limit locally. + +```toml +# sqlbuild_project.toml +[rules.thresholds] +max_literal_length = 100 # default +max_ranking_order_by = 5 # default +``` + +- `SQBRSQL042`: every output column of a non-final CTE must be read later in the same model, + in any clause of a later CTE or the final query, including through a later `SELECT *` + pass-through. The last CTE (the model output), unreachable CTEs (reported by `SQBRSQL005`), + CTEs with a column list, recursive queries and SQL-test fixtures are exempt. A later step that + uses an `@` interpolation in an expression may read any column, so it keeps every column, and + interpolated outputs are never reported. A later step that reads the CTE as a whole row (the + alias itself, as in `HASH(g)` or `COUNT(DISTINCT g)`), through a star nested in an expression + (`HASH(g.*)`), through `COLUMNS(...)`, or through `SELECT DISTINCT *` or a grouped `SELECT *` + reads every column. An output alias reused in the CTE's own `WHERE`, `GROUP BY`, `HAVING`, + `QUALIFY` or `ORDER BY` is read. A `GROUP BY` key can be dropped from the output while it stays + in `GROUP BY`. + `sqb format --fix` removes an unused column only when the removal keeps every row: the column + must be a column reference or a deterministic scalar expression (no aggregate, window, + set-returning or table function, subquery, or volatile function), and the CTE must not use + `SELECT DISTINCT`, `DISTINCT ON`, `GROUP BY ALL`, `ORDER BY ALL`, an aggregate without + `GROUP BY`, or positional `GROUP BY`/`ORDER BY`. Other findings are reported without a fix and + name the reason. The fix repeats until nothing changes. +- `SQBRSQL043`: `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `FIRST_VALUE` and `LAST_VALUE` windows may + sort by at most `max_ranking_order_by` expressions. Order by the one or two columns that decide + which row wins, then a unique row key as the final tie-breaker. If the rows have no unique + column, build one in the upstream model from the columns that identify a row (for example + `HASH(source_partition, source_offset, line_index) AS row_key`) and declare it with a `unique` + audit, so `SQBRSQL018` can prove the order. Raise the project limit only when the order + genuinely needs more keys. +- `SQBRSQL044`: a string literal longer than `max_literal_length` characters is an error. Define + the value once as a [constant](/docs/concepts/constants) in the narrowest + [scope](/docs/concepts/declaration-scopes) that covers every use, and reference it with + `@const`: a model-private `constants (_status_pattern '...')` entry in the `MODEL` header when + only that model uses it; a `CONSTANT (name status_pattern, value '...');` file in + `_sqlbuild/_constants/` (that folder only) or `_sqlbuild/constants/` (that folder and below) of + the nearest folder holding every use; top-level `constants/` only for project-wide values. + Otherwise shorten the literal. When the identical literal is reported in other files, the help + lists them (up to five) and names the nearest common folder and its constants directory. + Literals produced by `@const` or a macro are not reported, and there is no automatic fix. +- `SQBRSQL018` requires every `ROW_NUMBER`, `FIRST_VALUE` and `LAST_VALUE` window to be proven + deterministic: `PARTITION BY` and `ORDER BY` together must cover a unique key of the ranked rows. + Keys come from `unique` column audits and model-level `unique` audits naming a column, when the + column is also non-null (a `not_null` audit or `nullable false`), and from the `unique_key` of + incremental `merge` models and of `delete_insert` models matched by key without a cursor. + Snapshot, append, microbatch and cursor-ranged `delete_insert` `unique_key`s do not prove one + row per key. Keys are followed through projections, renames, filters, one-to-one joins, + `GROUP BY` grains (named or positional), `SELECT DISTINCT`, `UNION`, and + `ROW_NUMBER() ... = 1` deduplication in `QUALIFY` or a later `WHERE`; a `QUALIFY` key holds for + later steps, never for the window that defines it. A join that can match a row more than once, + `UNION ALL`, and relations without a declared key lose the proof, and "cannot prove" is an + error. To comply, declare the key upstream (for example + `columns (order_id (audits [unique, not_null]))`), or end the order with a unique + tie-breaker, building a row key upstream when none exists. `RANK` and + `DENSE_RANK` give tied rows the same value, so they need no proof. + +A finding that `sqb format --fix` can fix says so in text output and carries `"fixable": true` in +`sqb rules --json run` output; `sqb compile` adds the same note to the diagnostic. + Family selectors such as `select = ["SQBRSQL", "SQBRMODEL", "SQBRGRAPH"]` pick up new built-in Rules -on upgrade, so a release that adds `SQBRSQL040` and `SQBRSQL041` can expose computed join keys or -differently named final CTEs that were accepted before. +on upgrade, so a release that adds `SQBRSQL040` to `SQBRSQL044` can expose computed join keys, +differently named final CTEs, unused CTE columns, long literals or ranking sorts that were accepted +before. diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx index ae2b4359bd..8ead338596 100644 --- a/website/src/content/docs/docs/concepts/semantic-compilation.mdx +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -41,6 +41,18 @@ Known output shapes also validate `unique_key`, `cursor`, `cursor_type`, `cursor `config.partition_column`, `row_diff_exclude_columns`, `row_diff_tolerances.by_column`, source `cursor_column`, audit expressions/relationships/accepted values, and SQL-test mock/expected names. +The same binding, function-name and type checks run on every SQL body a project owns, not only +model queries: singular and generic audits (including measurement and evidence queries), SQL unit +tests (mock, helper, expected and assertion CTEs), and inline and named SQL hooks. An audit reading +`__ref("orders")` is checked against the closed `orders` output shape, so `WHERE o.shipped_at IS +NULL` on a model without that column fails compile instead of the warehouse. Each diagnostic names +its owner: `SQL test 'orders_are_open': ...`, `audit 'recent_rows' on model 'orders': ...` (with the +attached column when there is one), or `post_hooks[1] SQL hook: ...` at the hook file. Type +mismatches show the explicit conversion to write, such as `CAST('1' AS NUMBER)` in a test's expected +rows or `TO_DATE(@date_column)` in a generic audit. An +audit or SQL-test body the SQL parser cannot read is a P001 syntax error, never a silent pass. +Relations without a closed shape, such as physical tables written in hooks, stay open. + Expression type checks are enabled for **DuckDB (including MotherDuck), PostgreSQL, Snowflake, and BigQuery**. Other adapters retain reference and semantic checks without expression type checks. @@ -85,6 +97,9 @@ diagnostic warning count. Escape hatches are `MODEL (sql_analysis false)`, the corresponding path default, and `--no-sql-analysis`. Disabling SQL analysis also disables inference and lineage for that model. +`[settings] require_sql_analysis = true` limits the MODEL and path-default opt-outs to SQL the +parser cannot read; see [Requiring SQL analysis](/docs/concepts/models/configuration#requiring-sql-analysis). +Help that names a setting always shows the exact file, section and `key = value` to write. ## Reading diagnostics @@ -192,3 +207,6 @@ for example `order_id = CAST(customer_code AS INTEGER)`, or an explicit comparis B002–B005 predate these changes and overlap the executor's separately phased build codes. New semantic codes use distinct ranges. Authored model diagnostics retain source paths and positions. +Cast, coercible-literal and set-operation findings point at the offending cast, literal or branch +projection, and every finding is reported: two failing casts in one query are two diagnostics at +their own locations, never one merged diagnostic at the statement start. diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index c8d97f449e..4c3c64626a 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -665,6 +665,14 @@ tests/ ... ``` +## Skipping SQL analysis + +Compile checks SQL test fixtures and assertions like model SQL. For SQL the parser cannot read, +add `sql_analysis false` to the `TEST` header; the test still runs. Tests of a model with +`sql_analysis false`, and every test in a `--no-sql-analysis` run, are not analysed. With +`[settings] require_sql_analysis = true`, the opt-out is accepted only on test SQL that cannot be +parsed (see [Requiring SQL analysis](/docs/concepts/models/configuration#requiring-sql-analysis)). + ## Running tests Tests run automatically during `sqb build` in DAG order, before their target model is materialized.