diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/columns.rs similarity index 97% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/columns.rs index 54fe8f9b87..8a242d2165 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/columns.rs @@ -2,7 +2,7 @@ use std::cmp::Ordering; -use crate::semantic_checks::_helpers::text::{ascii, casefold}; +use crate::semantic_checks::_helpers::sql_text::text::{ascii, casefold}; use crate::semantic_checks::constants::{MAX_EDIT_DISTANCE, MIN_ABBREVIATION_LENGTH}; use crate::semantic_checks::models::SemanticDeferral; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/completion.rs similarity index 96% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/completion.rs index a19d501859..2c08a9399c 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/completion.rs @@ -5,19 +5,21 @@ use std::sync::LazyLock; use regex::Regex; -use crate::semantic_checks::_helpers::bindings::{BindingOwner, binding_positions}; -use crate::semantic_checks::_helpers::columns::closest_column; -use crate::semantic_checks::_helpers::explain::{ +use crate::semantic_checks::_helpers::explanation::columns::closest_column; +use crate::semantic_checks::_helpers::explanation::explain::{ Binary, ExplainContext, ModelEvidence, Shapes, binary_index, explain_model, is_explained_code, is_type_code, }; -use crate::semantic_checks::_helpers::messages::{ +use crate::semantic_checks::_helpers::explanation::messages::{ compiled, missing_column, pattern, semantic_help, sentence_message, }; -use crate::semantic_checks::_helpers::opt_outs::opt_out_diagnostic; -use crate::semantic_checks::_helpers::parsed_sql::{ParsedModelFacts, parsed_model_facts}; -use crate::semantic_checks::_helpers::poison::Poison; -use crate::semantic_checks::_helpers::text::{LineIndex, ascii, plain_text}; +use crate::semantic_checks::_helpers::explanation::opt_outs::opt_out_diagnostic; +use crate::semantic_checks::_helpers::recovery::bindings::{BindingOwner, binding_positions}; +use crate::semantic_checks::_helpers::recovery::poison::Poison; +use crate::semantic_checks::_helpers::sql_text::parsed_sql::{ + ParsedModelFacts, parsed_model_facts, +}; +use crate::semantic_checks::_helpers::sql_text::text::{LineIndex, ascii, plain_text}; use crate::semantic_checks::constants::{ MODEL_RESOURCE_TYPE, SEMANTIC_CODE_PREFIX, SQL_TEST_COLUMN_CODE, SQL_TEST_COLUMN_PATTERN, UNKNOWN_COLUMN_CODE, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/explain.rs similarity index 98% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/explain.rs index 8cf8c9f251..f6d3eaa26d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/explain.rs @@ -5,11 +5,11 @@ use std::sync::LazyLock; use regex::Regex; -use crate::semantic_checks::_helpers::columns::{closest_column, ordered_columns}; -use crate::semantic_checks::_helpers::messages::{ +use crate::semantic_checks::_helpers::explanation::columns::{closest_column, ordered_columns}; +use crate::semantic_checks::_helpers::explanation::messages::{ comparison_help, compiled, missing_column, pattern, semantic_help, sentence_message, type_words, }; -use crate::semantic_checks::_helpers::text::{ +use crate::semantic_checks::_helpers::sql_text::text::{ LineIndex, ascii, last_newline_before, newlines_before, prefix, }; use crate::semantic_checks::constants::{ diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/messages.rs similarity index 98% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/messages.rs index 770dd15a7f..7cb0732827 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/messages.rs @@ -4,7 +4,7 @@ use std::sync::LazyLock; use regex::{Captures, Regex}; -use crate::semantic_checks::_helpers::text::ascii; +use crate::semantic_checks::_helpers::sql_text::text::ascii; use crate::semantic_checks::constants::{ BIGQUERY_DIALECT, COMPARISON_CODE, CONTEXT_SUFFIX_PATTERN, DATE_TYPE, MISSING_PATTERN, QUOTED_PIECE_PATTERN, SEMANTIC_HELP, TIMESTAMP_TYPE, TYPE_WORDS_PATTERN, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/mod.rs new file mode 100644 index 0000000000..35eaafcc4e --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod columns; +pub(crate) mod completion; +pub(crate) mod explain; +pub(crate) mod messages; +pub(crate) mod opt_outs; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs similarity index 97% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs index 87d73a9db9..23a2ad7384 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs @@ -4,7 +4,7 @@ use std::sync::LazyLock; use regex::Regex; -use crate::semantic_checks::_helpers::messages::{compiled, pattern}; +use crate::semantic_checks::_helpers::explanation::messages::{compiled, pattern}; use crate::semantic_checks::constants::{ ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, PROJECT_CONFIG_FILENAME, REQUIRE_SQL_ANALYSIS_KEY, SETTING_SNIPPET_INDENT, SETTINGS_SECTION, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/argument_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/argument_types.rs new file mode 100644 index 0000000000..affd4eb433 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/argument_types.rs @@ -0,0 +1,92 @@ +//! Python's `_polyglot_expression_type` for one function argument expression. + +use std::collections::HashMap; + +use polyglot_sql::{DataType, Expression}; +use serde_json::Value; + +use crate::semantic_checks::_helpers::sql_text::text::ascii; +use crate::semantic_checks::constants::{ + BOOLEAN_RESULT_KINDS, BOOLEAN_TYPE_NAME, COLUMN_KIND, CUSTOM_TYPE_NAME, DECIMAL_TYPE_NAME, + KNOWN_CAST_TYPE_NAMES, TIMESTAMP_KIND, TIMESTAMP_WITH_TIME_ZONE_TYPE_NAME, VARCHAR_DATA_TYPES, +}; +use crate::semantic_checks::models::SemanticDeferral; + +/// The type Python infers for `expression`, or None. +pub(crate) fn expression_type( + expression: &Expression, + return_types: &HashMap, +) -> Result, SemanticDeferral> { + let kind: &str = expression.variant_name(); + let name: &str = expression.get_name(); + let function_name: &str = if name.is_empty() { kind } else { name }; + if kind != COLUMN_KIND + && let Some(declared) = return_types.get(&ascii(function_name)?.to_ascii_uppercase()) + { + return Ok(Some(declared.clone())); + } + if BOOLEAN_RESULT_KINDS.contains(&kind) { + return Ok(Some(BOOLEAN_TYPE_NAME.to_owned())); + } + match expression { + Expression::Cast(cast) | Expression::TryCast(cast) => cast_type(&cast.to), + _ => Ok(None), + } +} + +/// Python's reading of a cast's serialised `to` payload. +fn cast_type(data_type: &DataType) -> Result, SemanticDeferral> { + let target: Value = + serde_json::to_value(data_type).map_err(|_| SemanticDeferral::NativeFailure)?; + let Some(raw_type) = target + .get("data_type") + .and_then(Value::as_str) + .filter(|raw| !raw.is_empty()) + else { + return Ok(None); + }; + let raw_type: &str = ascii(raw_type)?; + let custom_name: Option<&str> = target + .get("name") + .and_then(Value::as_str) + .filter(|name| !name.is_empty()); + if raw_type.eq_ignore_ascii_case(CUSTOM_TYPE_NAME) + && let Some(name) = custom_name + { + return Ok(Some(ascii(name)?.to_ascii_uppercase())); + } + let lowered: String = raw_type.to_ascii_lowercase(); + if lowered == TIMESTAMP_KIND && target.get("timezone") == Some(&Value::Bool(true)) { + return Ok(Some(TIMESTAMP_WITH_TIME_ZONE_TYPE_NAME.to_owned())); + } + let type_name: String = KNOWN_CAST_TYPE_NAMES + .iter() + .find(|(raw, _)| *raw == raw_type) + .map_or_else( + || raw_type.replace('_', " ").to_ascii_uppercase(), + |(_, known)| (*known).to_owned(), + ); + if VARCHAR_DATA_TYPES.contains(&lowered.as_str()) + && let Some(length) = integer(&target, "length") + { + return Ok(Some(format!("VARCHAR({length})"))); + } + if type_name == DECIMAL_TYPE_NAME + && let Some(precision) = integer(&target, "precision") + { + return Ok(Some(match integer(&target, "scale") { + Some(scale) => format!("DECIMAL({precision}, {scale})"), + None => format!("DECIMAL({precision})"), + })); + } + Ok(Some(type_name)) +} + +/// Python's `isinstance(target.get(key), int)` value. +fn integer(target: &Value, key: &str) -> Option { + let value: &Value = target.get(key)?; + value + .as_i64() + .map(i128::from) + .or_else(|| value.as_u64().map(i128::from)) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs new file mode 100644 index 0000000000..b299cb02b6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs @@ -0,0 +1,271 @@ +//! Python's `get_semantic_metadata_diagnostics`, apart from the audit and resource SQL checks. + +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::semantic_checks::_helpers::explanation::messages::{compiled, pattern}; +use crate::semantic_checks::_helpers::metadata_checks::families::Families; +use crate::semantic_checks::_helpers::metadata_checks::function_calls::{ + FunctionContext, function_errors, +}; +use crate::semantic_checks::_helpers::metadata_checks::positions::{ + find_code_point, text_position, +}; +use crate::semantic_checks::_helpers::sql_text::text::{ascii, casefold}; +use crate::semantic_checks::constants::{ + CURSOR_INPUTS_KEY, SQL_TEST_COLUMN_CODE, SQL_TEST_CTE_PATTERN, TYPE_MISMATCH_CODE, + UNKNOWN_COLUMN_REFERENCE_CODE, UNKNOWN_TYPE, +}; +use crate::semantic_checks::models::{ + MetadataFinding, MetadataFunction, MetadataModel, MetadataOutcome, MetadataRequest, + MetadataSource, MetadataSqlTest, ModelMetadataFindings, SemanticDeferral, +}; +use crate::type_system::models::TypeFamily; + +static SQL_TEST_CTE: LazyLock> = + LazyLock::new(|| compiled(SQL_TEST_CTE_PATTERN)); + +const INTEGER_CURSOR_FAMILIES: &[TypeFamily] = &[TypeFamily::Integer, TypeFamily::Decimal]; +const TEMPORAL_CURSOR_FAMILIES: &[TypeFamily] = &[ + TypeFamily::Timestamp, + TypeFamily::Datetime, + TypeFamily::Date, +]; + +type Shapes<'a> = HashMap<&'a str, &'a [(String, String)]>; +/// One config reference: `(key, column name, the shape it must name a column of)`. +type Reference<'a> = (String, &'a str, &'a [(String, String)]); + +/// The metadata errors of every model, source and SQL test, or a deferral. +pub(crate) fn check_metadata( + request: &MetadataRequest, +) -> Result { + let shapes: Shapes<'_> = request + .shapes + .iter() + .map(|(name, columns)| (name.as_str(), columns.as_slice())) + .collect(); + let functions: HashMap<&str, &MetadataFunction> = request + .functions + .iter() + .map(|function| (function.key.as_str(), function)) + .collect(); + let return_types: HashMap = request.return_types.iter().cloned().collect(); + let families = Families::new(request.dialect.as_deref()); + let context = FunctionContext { + dialect: request.dialect.as_deref(), + functions: &functions, + shapes: &shapes, + return_types: &return_types, + families: &families, + }; + let mut models: Vec = Vec::with_capacity(request.models.len()); + for model in &request.models { + models.push(model_errors(model, &context)?); + } + let sources = source_errors(&request.sources, &shapes)?; + let sql_tests = sql_test_errors(&request.sql_tests, &shapes)?; + Ok(MetadataOutcome { + models, + sources, + sql_tests, + fallback_types: families.fallback_types(), + }) +} + +fn model_errors( + model: &MetadataModel, + context: &FunctionContext<'_>, +) -> Result { + if !model.checked { + return Ok(ModelMetadataFindings::default()); + } + let mut functions: Vec = Vec::new(); + for (code, name, message) in function_errors(model, context)? { + functions.push(model_error(model, code, &name, message)?); + } + let references = match context.shapes.get(model.name.as_str()) { + Some(shape) => reference_errors(model, shape, context)?, + None => Vec::new(), + }; + Ok(ModelMetadataFindings { + functions, + references, + }) +} + +/// Python's `_model_error`, located at the name's first whole-word use in authored SQL. +fn model_error( + model: &MetadataModel, + code: &'static str, + name: &str, + message: String, +) -> Result { + let (line, column) = text_position(&model.authored_sql, name, 0)?; + Ok(MetadataFinding { + code, + message, + line, + column, + }) +} + +/// Python's `{column.casefold() for column in shape}`. +fn folded_names(shape: &[(String, String)]) -> Result, SemanticDeferral> { + let mut names: HashSet = HashSet::with_capacity(shape.len()); + for (name, _) in shape { + names.insert(casefold(ascii(name)?)); + } + Ok(names) +} + +/// The B300 config references and the B301 cursor type of one model with a closed shape. +fn reference_errors( + model: &MetadataModel, + shape: &[(String, String)], + context: &FunctionContext<'_>, +) -> Result, SemanticDeferral> { + let mut references: Vec> = Vec::new(); + for (key, names) in &model.references { + for name in names { + references.push((key.clone(), name, shape)); + } + } + for (upstream, names) in &model.cursor_inputs { + if let Some(upstream_shape) = context.shapes.get(upstream.as_str()) { + for name in names { + references.push(( + format!("{CURSOR_INPUTS_KEY} {upstream}"), + name, + upstream_shape, + )); + } + } + } + let mut errors: Vec = Vec::new(); + for (key, name, reference_shape) in references { + if !folded_names(reference_shape)?.contains(&casefold(ascii(name)?)) { + let message = format!("{key} references unknown column '{name}'"); + errors.push(model_error( + model, + UNKNOWN_COLUMN_REFERENCE_CODE, + name, + message, + )?); + } + } + if let Some(error) = cursor_error(model, shape, context.families)? { + errors.push(error); + } + Ok(errors) +} + +/// Python's cursor-type check: the cursor column's family must suit `cursor_type`. +fn cursor_error( + model: &MetadataModel, + shape: &[(String, String)], + families: &Families, +) -> Result, SemanticDeferral> { + let (Some(cursor), Some(cursor_type)) = (model.cursor.as_deref(), model.cursor_type.as_deref()) + else { + return Ok(None); + }; + let folded_cursor: String = casefold(ascii(cursor)?); + let mut actual: &str = UNKNOWN_TYPE; + for (name, column_type) in shape { + if casefold(ascii(name)?) == folded_cursor { + actual = column_type; + break; + } + } + let allowed: &[TypeFamily] = match ascii(cursor_type)?.to_ascii_lowercase().as_str() { + "integer" => INTEGER_CURSOR_FAMILIES, + "timestamp" | "date" => TEMPORAL_CURSOR_FAMILIES, + _ => return Ok(None), + }; + let family: TypeFamily = families.family(actual)?; + if family == TypeFamily::Other || allowed.contains(&family) { + return Ok(None); + } + let message = + format!("cursor_type {cursor_type} does not match column '{cursor}' type {actual}"); + Ok(Some(model_error( + model, + TYPE_MISMATCH_CODE, + cursor, + message, + )?)) +} + +/// Python's source `cursor_column` check against each closed source shape. +fn source_errors( + sources: &[MetadataSource], + shapes: &Shapes<'_>, +) -> Result, SemanticDeferral> { + let mut errors: Vec<(usize, MetadataFinding)> = Vec::new(); + for (index, source) in sources.iter().enumerate() { + let (Some(cursor), Some(shape)) = (&source.cursor_column, shapes.get(source.name.as_str())) + else { + continue; + }; + if folded_names(shape)?.contains(&casefold(ascii(cursor)?)) { + continue; + } + let (line, column) = text_position(&source.contents, cursor, 0)?; + errors.push(( + index, + MetadataFinding { + code: UNKNOWN_COLUMN_REFERENCE_CODE, + message: format!("cursor_column references unknown column '{cursor}'"), + line, + column, + }, + )); + } + Ok(errors) +} + +/// Python's `_sql_test_errors`: B302 for fixture or expected columns a tested shape lacks. +fn sql_test_errors( + tests: &[MetadataSqlTest], + shapes: &Shapes<'_>, +) -> Result, SemanticDeferral> { + let cte_pattern: &Regex = pattern(&SQL_TEST_CTE)?; + let mut errors: Vec<(usize, MetadataFinding, i64)> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + for (cte_name, columns) in &test.ctes { + if cte_name.ends_with('\n') { + return Err(SemanticDeferral::NonAsciiText); + } + let Some(shape) = cte_pattern + .captures(cte_name) + .and_then(|captures| shapes.get(&captures[1])) + else { + continue; + }; + let available: HashSet = folded_names(shape)?; + for column_name in columns { + if available.contains(&casefold(ascii(column_name)?)) { + continue; + } + let offset: usize = find_code_point(&test.contents, cte_name).unwrap_or(0); + let (line, column) = text_position(&test.contents, column_name, offset)?; + let width: i64 = i64::try_from(column_name.chars().count()).unwrap_or(i64::MAX); + let message = format!("SQL test '{cte_name}' names unknown column '{column_name}'"); + errors.push(( + index, + MetadataFinding { + code: SQL_TEST_COLUMN_CODE, + message, + line, + column, + }, + column + width, + )); + } + } + } + Ok(errors) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/families.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/families.rs new file mode 100644 index 0000000000..c63fa668ec --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/families.rs @@ -0,0 +1,61 @@ +//! Python's `normalize_type(...).family` through the native type system, noting text fallbacks. + +use std::cell::RefCell; + +use crate::semantic_checks::constants::GENERIC_DIALECT; +use crate::semantic_checks::models::SemanticDeferral; +use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::models::{TypeFamily, TypeNormalization}; + +const NUMERIC_FAMILIES: [TypeFamily; 3] = + [TypeFamily::Integer, TypeFamily::Decimal, TypeFamily::Float]; +const TEMPORAL_FAMILIES: [TypeFamily; 3] = [ + TypeFamily::Timestamp, + TypeFamily::Date, + TypeFamily::Datetime, +]; + +/// Type families for one dialect, remembering types Python logs a parse fallback for. +pub(crate) struct Families { + dialect: String, + fallback_types: RefCell>, +} + +impl Families { + pub(crate) fn new(dialect: Option<&str>) -> Self { + Self { + dialect: dialect.unwrap_or(GENERIC_DIALECT).to_owned(), + fallback_types: RefCell::new(Vec::new()), + } + } + + /// The family of `type_sql`, or a deferral where Python would leave the native type system. + pub(crate) fn family(&self, type_sql: &str) -> Result { + let normalization: TypeNormalization = + normalize_type(type_sql, &self.dialect).ok_or(SemanticDeferral::UnsupportedType)?; + if normalization.parse_error.is_some() { + self.fallback_types.borrow_mut().push(type_sql.to_owned()); + } + Ok(normalization.normalized.family) + } + + /// Python's `_different_argument_families`. + pub(crate) fn different(&self, left: &str, right: &str) -> Result { + let left: TypeFamily = self.family(left)?; + let right: TypeFamily = self.family(right)?; + let families: [TypeFamily; 2] = [left, right]; + Ok(left != right + && !families.contains(&TypeFamily::Other) + && !families + .iter() + .all(|family| NUMERIC_FAMILIES.contains(family)) + && !families + .iter() + .all(|family| TEMPORAL_FAMILIES.contains(family))) + } + + /// Types whose normalization fell back to text, in the order they were normalized. + pub(crate) fn fallback_types(self) -> Vec { + self.fallback_types.into_inner() + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/function_calls.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/function_calls.rs new file mode 100644 index 0000000000..1c9bc2ec0b --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/function_calls.rs @@ -0,0 +1,245 @@ +//! Python's `_function_errors`: argument counts and families of declared function calls. + +use std::collections::HashMap; + +use polyglot_sql::expressions::{Column, Function, Literal, Select}; +use polyglot_sql::{Expression, ExpressionWalk}; + +use crate::semantic_checks::_helpers::metadata_checks::argument_types::expression_type; +use crate::semantic_checks::_helpers::metadata_checks::families::Families; +use crate::semantic_checks::_helpers::sql_text::parsed_sql::parsed_model; +use crate::semantic_checks::_helpers::sql_text::text::{ascii, casefold}; +use crate::semantic_checks::constants::{ + ARGUMENT_COUNT_CODE, NUMBER_LITERAL_TYPE, STRING_LITERAL_TYPE, TYPE_MISMATCH_CODE, + UDF_NAME_PREFIX, UNKNOWN_TYPE, +}; +use crate::semantic_checks::models::{MetadataFunction, MetadataModel, SemanticDeferral}; + +/// One function error before it is located: `(code, located name, message)`. +pub(crate) type FunctionError = (&'static str, String, String); + +/// Everything `_function_errors` reads beside the model. +pub(crate) struct FunctionContext<'a> { + pub(crate) dialect: Option<&'a str>, + pub(crate) functions: &'a HashMap<&'a str, &'a MetadataFunction>, + pub(crate) shapes: &'a HashMap<&'a str, &'a [(String, String)]>, + pub(crate) return_types: &'a HashMap, + pub(crate) families: &'a Families, +} + +/// One declared function call, the index of its nearest select, and its declaration. +struct DeclaredCall<'a> { + call: &'a Function, + select: Option, + function: &'a MetadataFunction, +} + +/// Case-folded column types of one relation shape, preserving declaration order per name. +type FoldedShape = HashMap>; + +/// The B102 and B301 errors of one model's declared function calls. +pub(crate) fn function_errors( + model: &MetadataModel, + context: &FunctionContext<'_>, +) -> Result, SemanticDeferral> { + if context.functions.is_empty() || !model.calls_functions { + return Ok(Vec::new()); + } + let Some(parsed) = parsed_model(&model.query_sql, context.dialect)? else { + return Ok(Vec::new()); + }; + let (calls, selects) = declared_calls(&parsed, context.functions)?; + let mut relations: HashMap> = HashMap::new(); + for call in &calls { + if let Some(select) = call.select + && resolves_argument_columns(call) + { + relations + .entry(select) + .or_insert_with(|| select_relations(selects[select])); + } + } + let folded: HashMap<&str, FoldedShape> = folded_shapes(&relations, context.shapes)?; + let mut errors: Vec = Vec::new(); + for call in &calls { + let name: String = call_name(call.call).to_owned(); + let declared: &[(String, String)] = &call.function.arguments; + if call.call.args.len() != declared.len() { + let message = format!( + "Function '{name}' expects {} arguments but received {}", + declared.len(), + call.call.args.len() + ); + errors.push((ARGUMENT_COUNT_CODE, name, message)); + continue; + } + let scope: Option<&[&Expression]> = call + .select + .and_then(|select| relations.get(&select)) + .map(Vec::as_slice); + for (argument, (argument_name, argument_type)) in call.call.args.iter().zip(declared) { + let actual: String = argument_type_of(argument, scope, &folded, context)?; + if actual != UNKNOWN_TYPE && context.families.different(&actual, argument_type)? { + let message = format!( + "Function '{name}' argument '{argument_name}' expects {argument_type}, \ + received {actual}; convert the argument explicitly" + ); + errors.push((TYPE_MISMATCH_CODE, name.clone(), message)); + } + } + } + Ok(errors) +} + +fn call_name(call: &Function) -> &str { + call.name + .strip_prefix(UDF_NAME_PREFIX) + .unwrap_or(&call.name) +} + +/// Python's `_declared_function_calls` over `_function_scopes`' pre-order walk. +fn declared_calls<'a>( + root: &'a Expression, + functions: &HashMap<&str, &'a MetadataFunction>, +) -> Result<(Vec>, Vec<&'a Select>), SemanticDeferral> { + let mut calls: Vec> = Vec::new(); + let mut selects: Vec<&'a Select> = Vec::new(); + let mut pending: Vec<(&'a Expression, Option)> = vec![(root, None)]; + while let Some((node, mut scope)) = pending.pop() { + match node { + Expression::Select(select) => { + scope = Some(selects.len()); + selects.push(select); + } + Expression::Function(call) => { + let key: String = casefold(ascii(call_name(call))?); + if let Some(function) = functions.get(key.as_str()) { + calls.push(DeclaredCall { + call, + select: scope, + function, + }); + } + } + _ => {} + } + for child in node.children().into_iter().rev() { + pending.push((child, scope)); + } + } + Ok((calls, selects)) +} + +/// Python's `_resolves_argument_columns`. +fn resolves_argument_columns(call: &DeclaredCall<'_>) -> bool { + call.call.args.len() == call.function.arguments.len() + && call + .call + .args + .iter() + .any(|argument| matches!(argument, Expression::Column(_))) +} + +/// Python's `_select_relations`: FROM expressions, then each join's relation. +fn select_relations(select: &Select) -> Vec<&Expression> { + let mut relations: Vec<&Expression> = select + .from + .as_ref() + .map(|from| from.expressions.iter().collect()) + .unwrap_or_default(); + relations.extend(select.joins.iter().map(|join| &join.this)); + relations +} + +/// Python's `_relation_table_name` and the alias `_argument_column_type` compares. +fn relation_names(relation: &Expression) -> (&str, &str) { + match relation { + Expression::Table(table) => ( + table.name.name.as_str(), + table + .alias + .as_ref() + .map_or(table.name.name.as_str(), |alias| alias.name.as_str()), + ), + _ => ("", ""), + } +} + +/// Python's `folded_shapes` for the tables the resolving selects read. +fn folded_shapes<'a>( + relations: &HashMap>, + shapes: &HashMap<&str, &[(String, String)]>, +) -> Result, SemanticDeferral> { + let mut folded: HashMap<&'a str, FoldedShape> = HashMap::new(); + for relation in relations.values().flatten() { + let (table_name, _) = relation_names(relation); + let Some(shape) = shapes.get(table_name) else { + continue; + }; + if folded.contains_key(table_name) { + continue; + } + let mut columns: FoldedShape = HashMap::new(); + for (name, column_type) in *shape { + columns + .entry(casefold(ascii(name)?)) + .or_default() + .push(column_type.clone()); + } + folded.insert(table_name, columns); + } + Ok(folded) +} + +/// The type `_function_errors` compares for one argument. +fn argument_type_of( + argument: &Expression, + scope: Option<&[&Expression]>, + folded: &HashMap<&str, FoldedShape>, + context: &FunctionContext<'_>, +) -> Result { + match argument { + Expression::Literal(literal) => Ok(match literal.as_ref() { + Literal::String(_) => STRING_LITERAL_TYPE, + Literal::Number(_) => NUMBER_LITERAL_TYPE, + _ => UNKNOWN_TYPE, + } + .to_owned()), + Expression::Column(column) => argument_column_type(column, scope, folded), + _ => Ok(expression_type(argument, context.return_types)? + .unwrap_or_else(|| UNKNOWN_TYPE.to_owned())), + } +} + +/// Python's `_argument_column_type`: the only matching column type among the select's inputs. +fn argument_column_type( + column: &Column, + scope: Option<&[&Expression]>, + folded: &HashMap<&str, FoldedShape>, +) -> Result { + let Some(relations) = scope else { + return Ok(UNKNOWN_TYPE.to_owned()); + }; + let qualifier: Option = match &column.table { + Some(table) => Some(casefold(ascii(&table.name)?)), + None => None, + }; + let name: String = casefold(ascii(&column.name.name)?); + let mut candidates: Vec<&String> = Vec::new(); + for relation in relations { + let (table_name, alias) = relation_names(relation); + if let Some(qualifier) = &qualifier + && *qualifier != casefold(ascii(alias)?) + { + continue; + } + let Some(shape) = folded.get(table_name) else { + return Ok(UNKNOWN_TYPE.to_owned()); + }; + candidates.extend(shape.get(&name).into_iter().flatten()); + } + Ok(match candidates.as_slice() { + [only] => (*only).clone(), + _ => UNKNOWN_TYPE.to_owned(), + }) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs new file mode 100644 index 0000000000..4e52b92c32 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod argument_types; +pub(crate) mod checks; +pub(crate) mod families; +pub(crate) mod function_calls; +pub(crate) mod positions; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/positions.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/positions.rs new file mode 100644 index 0000000000..a18abde30d --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/positions.rs @@ -0,0 +1,78 @@ +//! Python's `_text_position`: the first whole-word occurrence of a name, in code points. + +use crate::semantic_checks::models::SemanticDeferral; + +/// Python's `\w` for a character next to a match; beyond ASCII the two engines may disagree. +fn word_character(character: char) -> Result { + if character.is_ascii() { + Ok(character.is_ascii_alphanumeric() || character == '_') + } else { + Err(SemanticDeferral::NonAsciiText) + } +} + +/// Python's `text.find(needle)` in code points, or None. +pub(crate) fn find_code_point(text: &str, needle: &str) -> Option { + text.find(needle).map(|byte| text[..byte].chars().count()) +} + +/// Whether `characters[start..]` holds `name` at `position` as a whole word. +fn whole_word_at( + characters: &[char], + name: &[char], + start: usize, + position: usize, +) -> Result { + if characters[position..position + name.len()] != *name { + return Ok(false); + } + if position > start && word_character(characters[position - 1])? { + return Ok(false); + } + match characters.get(position + name.len()) { + Some(next) => Ok(!word_character(*next)?), + None => Ok(true), + } +} + +/// Python's `re.search(rf"(? Result { + let start: usize = offset.min(characters.len()); + if name.len() <= characters.len() - start { + for position in start..=characters.len() - name.len() { + if whole_word_at(characters, name, start, position)? { + return Ok(position); + } + } + } + Ok(offset) +} + +/// Python's `_text_position`: the 1-based line and column of `name` at or after `offset`. +pub(crate) fn text_position( + text: &str, + name: &str, + offset: usize, +) -> Result<(i64, i64), SemanticDeferral> { + let characters: Vec = text.chars().collect(); + let name: Vec = name.chars().collect(); + let start: usize = match_start(&characters, &name, offset)?; + let before: &[char] = &characters[..start.min(characters.len())]; + let lines: usize = before + .iter() + .filter(|character| **character == '\n') + .count(); + let last_newline: i64 = before + .iter() + .rposition(|character| *character == '\n') + .map_or(-1, |index| i64::try_from(index).unwrap_or(i64::MAX)); + let start: i64 = i64::try_from(start).unwrap_or(i64::MAX); + Ok(( + i64::try_from(lines).unwrap_or(i64::MAX) + 1, + start - last_newline, + )) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs index 166cab73d1..9d1303099c 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs @@ -1,10 +1,4 @@ -pub(crate) mod bindings; -pub(crate) mod columns; -pub(crate) mod completion; -pub(crate) mod explain; -pub(crate) mod messages; -pub(crate) mod opt_outs; -pub(crate) mod parsed_sql; -pub(crate) mod poison; -pub(crate) mod text; -pub(crate) mod type_recovery; +pub(crate) mod explanation; +pub(crate) mod metadata_checks; +pub(crate) mod recovery; +pub(crate) mod sql_text; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/bindings.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/bindings.rs similarity index 100% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/bindings.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/bindings.rs diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/mod.rs new file mode 100644 index 0000000000..edb1c88983 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod bindings; +pub(crate) mod poison; +pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/poison.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/poison.rs similarity index 100% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/poison.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/poison.rs diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/type_recovery.rs similarity index 97% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/type_recovery.rs index 9e1446429b..225ef721ee 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/recovery/type_recovery.rs @@ -2,9 +2,9 @@ use std::collections::{HashMap, HashSet, VecDeque}; -use crate::semantic_checks::_helpers::bindings::{BindingOwner, binding_positions}; -use crate::semantic_checks::_helpers::parsed_sql::{normalized_sql, projection_spans}; -use crate::semantic_checks::_helpers::poison::Poison; +use crate::semantic_checks::_helpers::recovery::bindings::{BindingOwner, binding_positions}; +use crate::semantic_checks::_helpers::recovery::poison::Poison; +use crate::semantic_checks::_helpers::sql_text::parsed_sql::{normalized_sql, projection_spans}; use crate::semantic_checks::models::{ RecoveryModel, SemanticDeferral, TypeRecoveryOutcome, TypeRecoveryPlan, TypeRecoveryRequest, TypeRecoveryStep, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/mod.rs new file mode 100644 index 0000000000..4eeca97b60 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod parsed_sql; +pub(crate) mod text; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs similarity index 94% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs index 6fb68b7e1f..d70a47139a 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs @@ -118,6 +118,20 @@ pub(crate) fn parsed_model_facts( query_sql: &str, dialect: Option<&str>, ) -> Result { + let Some(parsed) = parsed_model(query_sql, dialect)? else { + return Ok(ParsedModelFacts::default()); + }; + Ok(ParsedModelFacts { + aliases: input_aliases(&parsed), + unaliased_outputs: unaliased_outputs(&parsed), + }) +} + +/// The wheel's guarded `parse_one` of normalized model SQL; None where it raises. +pub(crate) fn parsed_model( + query_sql: &str, + dialect: Option<&str>, +) -> Result, SemanticDeferral> { let sql = normalized_sql(query_sql, dialect)?; let parser = polyglot_dialect(dialect.unwrap_or(GENERIC_DIALECT))?; let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ @@ -128,16 +142,12 @@ pub(crate) fn parsed_model_facts( complexity_guard: Some(guard), }; let Ok(mut statements) = parser.parse_with_options(&sql, &options) else { - return Ok(ParsedModelFacts::default()); + return Ok(None); }; if statements.len() != 1 { - return Ok(ParsedModelFacts::default()); + return Ok(None); } - let parsed: Expression = statements.remove(0); - Ok(ParsedModelFacts { - aliases: input_aliases(&parsed), - unaliased_outputs: unaliased_outputs(&parsed), - }) + Ok(Some(statements.remove(0))) } fn input_aliases(parsed: &Expression) -> HashMap { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/text.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/text.rs similarity index 100% rename from crates/sqlbuild-analysis/src/semantic_checks/_helpers/text.rs rename to crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/text.rs diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index d42aef5b0c..d423aae445 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -24,6 +24,58 @@ pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; pub(crate) const TIMESTAMP_TYPE: &str = "TIMESTAMP"; pub(crate) const DATE_TYPE: &str = "DATE"; pub(crate) const BIGQUERY_DIALECT: &str = "bigquery"; +pub(crate) const UNKNOWN_TYPE: &str = "UNKNOWN"; +pub(crate) const UNKNOWN_COLUMN_REFERENCE_CODE: &str = "B300"; +pub(crate) const TYPE_MISMATCH_CODE: &str = "B301"; +pub(crate) const ARGUMENT_COUNT_CODE: &str = "B102"; +pub(crate) const UDF_NAME_PREFIX: &str = "__sqlbuild_udf_"; +pub(crate) const COLUMN_KIND: &str = "column"; +pub(crate) const TIMESTAMP_KIND: &str = "timestamp"; +pub(crate) const BOOLEAN_TYPE_NAME: &str = "BOOLEAN"; +pub(crate) const CUSTOM_TYPE_NAME: &str = "CUSTOM"; +pub(crate) const DECIMAL_TYPE_NAME: &str = "DECIMAL"; +pub(crate) const TIMESTAMP_WITH_TIME_ZONE_TYPE_NAME: &str = "TIMESTAMPTZ"; +pub(crate) const VARCHAR_DATA_TYPES: [&str; 2] = ["var_char", "varchar"]; +pub(crate) const STRING_LITERAL_TYPE: &str = "VARCHAR"; +pub(crate) const NUMBER_LITERAL_TYPE: &str = "DOUBLE"; +pub(crate) const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; +pub(crate) const SQL_TEST_CTE_PATTERN: &str = r"\A__(?:expected|ref|source|seed)__(.+)\z"; +pub(crate) const BOOLEAN_RESULT_KINDS: [&str; 18] = [ + "boolean", + "and", + "between", + "eq", + "exists", + "gt", + "gte", + "ilike", + "in", + "is", + "is_null", + "like", + "lt", + "lte", + "neq", + "not", + "or", + "regexp_like", +]; +/// Python's `_polyglot_type_name` table of serialised cast types. +pub(crate) const KNOWN_CAST_TYPE_NAMES: [(&str, &str); 13] = [ + ("big_int", "BIGINT"), + ("bool", "BOOLEAN"), + ("boolean", "BOOLEAN"), + ("date", "DATE"), + ("decimal", "DECIMAL"), + ("double", "DOUBLE"), + ("float", "FLOAT"), + ("int", "INT"), + ("integer", "INT"), + ("text", "TEXT"), + ("timestamp", "TIMESTAMP"), + ("var_char", "TEXT"), + ("varchar", "TEXT"), +]; pub(crate) const TYPE_WORDS_PATTERN: &str = r"(?i)\b(timestamp|integer|varchar|boolean|date|interval|double|float|decimal|numeric|bigint|smallint|text|time|string|binary|array|struct)\b"; pub(crate) const MISSING_PATTERN: &str = r"Unknown column '([^']+)'(?: in table '([^']+)')?"; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs new file mode 100644 index 0000000000..d49d847409 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs @@ -0,0 +1,19 @@ +//! Python's semantic metadata checks after type recovery, without the Polyglot wheel. + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::semantic_checks::_helpers::metadata_checks::checks::check_metadata; +use crate::semantic_checks::models::{MetadataOutcome, MetadataRequest, SemanticDeferral}; +use crate::semantic_validation::models::ProjectCatalog; + +/// Check metadata on the compile's analysis pool; a deferral hands the checks back to Python. +pub fn check_semantic_metadata( + request: &MetadataRequest, + catalog: &ProjectCatalog, +) -> Result, String> { + let pool = catalog.analysis_pool()?; + Ok(pool.install(|| { + catch_compiler_panic(|| Ok(check_metadata(request))) + .unwrap_or(Err(SemanticDeferral::NativeFailure)) + })) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs index ade232f46b..1d0f251c6b 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs @@ -2,7 +2,7 @@ use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::semantic_checks::_helpers::completion::complete; +use crate::semantic_checks::_helpers::explanation::completion::complete; use crate::semantic_checks::models::{CompletionOutcome, CompletionRequest, SemanticDeferral}; use crate::semantic_validation::models::ProjectCatalog; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs index 276e4fd3db..850bf1ae2c 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs @@ -2,7 +2,7 @@ use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::semantic_checks::_helpers::type_recovery::finish; +use crate::semantic_checks::_helpers::recovery::type_recovery::finish; use crate::semantic_checks::models::{TypeRecoveryOutcome, TypeRecoveryPlan, TypeRecoveryRequest}; use crate::semantic_checks::types::RevisedBinding; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs index 25aba0548b..9dd5e1a128 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs @@ -1,3 +1,4 @@ +pub mod check_semantic_metadata; pub mod complete_semantic_diagnostics; pub mod finish_type_recovery; pub mod plan_type_recovery; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs index 8ee38b7c1a..2cfab0de26 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs @@ -2,7 +2,7 @@ use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; -use crate::semantic_checks::_helpers::type_recovery::plan; +use crate::semantic_checks::_helpers::recovery::type_recovery::plan; use crate::semantic_checks::models::{SemanticDeferral, TypeRecoveryRequest, TypeRecoveryStep}; use crate::semantic_validation::models::ProjectCatalog; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs index 10cd62be2d..de89887eb4 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -66,6 +66,7 @@ pub enum SemanticDeferral { UnreadableSql, NonAsciiText, UnexpectedPosition, + UnsupportedType, NativeFailure, } @@ -77,6 +78,7 @@ impl SemanticDeferral { Self::UnreadableSql => "unreadable_sql", Self::NonAsciiText => "non_ascii_text", Self::UnexpectedPosition => "unexpected_position", + Self::UnsupportedType => "unsupported_type", Self::NativeFailure => "native_failure", } } @@ -262,3 +264,87 @@ impl CompletionOutcome { } } } + +/// One declared project function as Python's metadata checks read it. +#[derive(Clone, Debug)] +pub struct MetadataFunction { + /// Python's `function.name.casefold()`, the key calls are looked up by. + pub key: String, + /// `(argument name, declared type)` in declaration order. + pub arguments: Vec<(String, String)>, +} + +/// One model as Python's metadata checks read it. +#[derive(Clone, Debug)] +pub struct MetadataModel { + pub name: String, + pub query_sql: String, + pub authored_sql: String, + /// False when the model opts out of SQL analysis or was never binding-validated. + pub checked: bool, + /// Whether the model references a UDF or table function. + pub calls_functions: bool, + /// `(config key, column names)` checked against the model's own shape, in Python's order. + pub references: Vec<(String, Vec)>, + /// `(upstream, column names)` of `cursor_inputs`, in Python's dict order. + pub cursor_inputs: Vec<(String, Vec)>, + pub cursor: Option, + pub cursor_type: Option, +} + +/// One source's cursor column and file text. +#[derive(Clone, Debug)] +pub struct MetadataSource { + pub name: String, + /// The cursor column, when it is set and non-empty. + pub cursor_column: Option, + pub contents: String, +} + +/// One model SQL test's file text and its CTEs with their inferred column names. +#[derive(Clone, Debug)] +pub struct MetadataSqlTest { + pub contents: String, + pub ctes: Vec<(String, Vec)>, +} + +/// Python's `get_semantic_metadata_diagnostics` inputs, apart from audits and resource SQL. +#[derive(Clone, Debug)] +pub struct MetadataRequest { + pub dialect: Option, + /// The inference profile's declared function return types. + pub return_types: Vec<(String, String)>, + pub functions: Vec, + /// Closed relation shapes in Python's dict order. + pub shapes: Vec<(String, Vec<(String, String)>)>, + pub models: Vec, + pub sources: Vec, + pub sql_tests: Vec, +} + +/// One metadata error located in its resource's text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MetadataFinding { + pub code: &'static str, + pub message: String, + pub line: i64, + pub column: i64, +} + +/// One model's function errors, which precede its audits, and its config reference errors. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelMetadataFindings { + pub functions: Vec, + pub references: Vec, +} + +/// The metadata errors per resource, and the types whose normalization fell back to text. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MetadataOutcome { + pub models: Vec, + /// `(source index, error)`. + pub sources: Vec<(usize, MetadataFinding)>, + /// `(SQL test index, error, end column)`. + pub sql_tests: Vec<(usize, MetadataFinding, i64)>, + pub fallback_types: Vec, +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs index cda52abf5f..39f180ba86 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_argument_types.rs"] +mod argument_types; #[path = "tests/test_columns.rs"] mod columns; #[path = "tests/test_completion.rs"] @@ -6,10 +8,14 @@ mod completion; mod helpers; #[path = "tests/test_messages.rs"] mod messages; +#[path = "tests/test_metadata.rs"] +mod metadata; #[path = "tests/test_operand_types.rs"] mod operand_types; #[path = "tests/test_parsed_sql.rs"] mod parsed_sql; +#[path = "tests/test_positions.rs"] +mod positions; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_type_recovery.rs"] diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs index 37ee96f1c9..ab3f171b18 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs @@ -1,19 +1,24 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; -use polyglot_sql::{DialectType, SchemaValidationOptions}; +use polyglot_sql::{DialectType, ExpressionWalk, SchemaValidationOptions}; -use crate::semantic_checks::_helpers::messages::{missing_column, sentence_message}; -use crate::semantic_checks::_helpers::parsed_sql::parsed_model_facts; +use crate::semantic_checks::_helpers::explanation::messages::{missing_column, sentence_message}; +use crate::semantic_checks::_helpers::metadata_checks::argument_types::expression_type; +use crate::semantic_checks::_helpers::sql_text::parsed_sql::{parsed_model, parsed_model_facts}; +use crate::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use crate::semantic_checks::main::complete_semantic_diagnostics::complete_semantic_diagnostics; use crate::semantic_checks::main::finish_type_recovery::finish_type_recovery; use crate::semantic_checks::main::plan_type_recovery::plan_type_recovery; use crate::semantic_checks::models::{ CompletedDiagnostic, CompletionModel, CompletionRequest, DiagnosticOwner, FinalDiagnostic, - LineageOutput, ModelBinding, RawBinding, RecoveryModel, SemanticDeferral, SemanticDiagnostic, - SemanticLocation, TypeRecoveryPlan, TypeRecoveryRequest, + LineageOutput, MetadataFinding, MetadataFunction, MetadataModel, MetadataOutcome, + MetadataRequest, MetadataSource, MetadataSqlTest, ModelBinding, ModelMetadataFindings, + RawBinding, RecoveryModel, SemanticDeferral, SemanticDiagnostic, SemanticLocation, + TypeRecoveryPlan, TypeRecoveryRequest, }; use crate::semantic_checks::tests::test_types::{ - CompletionTestCase, DescribedDiagnostic, OperandTypeTestCase, TypeRecoveryTestCase, + CompletionTestCase, DescribedDiagnostic, MetadataTestCase, OperandTypeTestCase, + TypeRecoveryTestCase, }; use crate::semantic_validation::models::ProjectCatalog; @@ -497,3 +502,194 @@ pub(crate) fn operand_type_summary(test_case: &OperandTypeTestCase) -> Completio .collect(); (deferral, described_order, model_bindings) } + +/// The type Python infers for the first projection of `SELECT {expression}` on DuckDB. +pub(crate) fn projected_type(expression: &str) -> Option { + let parsed = parsed_model(&format!("SELECT {expression}"), Some("duckdb")) + .expect("duckdb parses") + .expect("the projection parses"); + let projection = parsed + .children() + .into_iter() + .next() + .expect("one projection"); + let return_types: HashMap = + HashMap::from([("MY_FN".to_owned(), "DATE".to_owned())]); + expression_type(projection, &return_types).expect("ASCII expressions are typed") +} + +fn metadata_model(name: &str, query_sql: &str, checked: bool) -> MetadataModel { + MetadataModel { + name: name.to_owned(), + query_sql: query_sql.to_owned(), + authored_sql: format!("MODEL (\n description \"{name}\",\n);\n\n{query_sql}\n"), + checked, + calls_functions: true, + references: vec![ + ("unique_key".to_owned(), strings(&["order_id"])), + ("cursor".to_owned(), strings(&["label"])), + ( + "row_diff_exclude_columns".to_owned(), + strings(&["missing_col"]), + ), + ], + cursor_inputs: vec![ + ( + "raw_orders".to_owned(), + strings(&["updated_at", "created_at"]), + ), + ("unknown_upstream".to_owned(), strings(&["x"])), + ], + cursor: Some("label".to_owned()), + cursor_type: Some("timestamp".to_owned()), + } +} + +/// A mart calling declared functions badly, with config, source and SQL test metadata. +pub(crate) fn metadata_request(test_case: &MetadataTestCase) -> MetadataRequest { + let query_sql = "SELECT __udf(\"scaled_amount\")(o.amount, 2) AS scaled,\n \ + __udf(\"scaled_amount\")(o.status, 'x') AS wrong,\n \ + __udf(\"label_status\")(o.status, 1) AS label\nFROM __ref(\"stg\") AS o"; + MetadataRequest { + dialect: Some(test_case.dialect.to_owned()), + return_types: Vec::new(), + functions: vec![ + MetadataFunction { + key: "scaled_amount".to_owned(), + arguments: shape(&[("p_amount", "DOUBLE"), ("p_factor", "INTEGER")]), + }, + MetadataFunction { + key: "label_status".to_owned(), + arguments: shape(&[("p_status", "VARCHAR")]), + }, + ], + shapes: vec![ + ( + "stg".to_owned(), + shape(&[ + ("order_id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), + ]), + ), + ( + "raw_orders".to_owned(), + shape(&[("order_id", "INTEGER"), ("updated_at", "TIMESTAMP")]), + ), + ("mart".to_owned(), shape(test_case.mart_shape)), + ], + models: vec![ + metadata_model("mart", query_sql, true), + metadata_model("skipped", query_sql, false), + ], + sources: vec![MetadataSource { + name: "raw_orders".to_owned(), + cursor_column: Some("loaded_at".to_owned()), + contents: "sources:\n - name: raw_orders\n cursor_column: loaded_at\n".to_owned(), + }], + sql_tests: vec![MetadataSqlTest { + contents: "-- test\nWITH __ref__stg AS (SELECT 1 AS order_id, 2 AS refund)\n" + .to_owned(), + ctes: vec![ + ("__ref__stg".to_owned(), strings(&["order_id", "refund"])), + ("__expected__mart".to_owned(), strings(&["scaled", "LABEL"])), + ("helper".to_owned(), strings(&["zzz"])), + ], + }], + } +} + +/// One expected located metadata error. +pub(crate) fn metadata_finding( + code: &'static str, + message: &str, + line: i64, + column: i64, +) -> MetadataFinding { + MetadataFinding { + code, + message: message.to_owned(), + line, + column, + } +} + +/// Run the metadata checks on a fresh catalog. +pub(crate) fn metadata_outcome( + test_case: &MetadataTestCase, +) -> Result { + check_semantic_metadata(&metadata_request(test_case), &catalog()).expect("the pool runs") +} + +/// The findings Python reports for the checked mart of `metadata_request`. +pub(crate) fn expected_checked_outcome() -> MetadataOutcome { + MetadataOutcome { + models: vec![ + ModelMetadataFindings { + functions: vec![ + metadata_finding( + "B301", + "Function 'scaled_amount' argument 'p_amount' expects DOUBLE, received \ + VARCHAR; convert the argument explicitly", + 5, + 15, + ), + metadata_finding( + "B301", + "Function 'scaled_amount' argument 'p_factor' expects INTEGER, received \ + VARCHAR; convert the argument explicitly", + 5, + 15, + ), + metadata_finding( + "B102", + "Function 'label_status' expects 1 arguments but received 2", + 7, + 10, + ), + ], + references: vec![ + metadata_finding( + "B300", + "row_diff_exclude_columns references unknown column 'missing_col'", + 1, + 1, + ), + metadata_finding( + "B300", + "cursor_inputs raw_orders references unknown column 'created_at'", + 1, + 1, + ), + metadata_finding( + "B301", + "cursor_type timestamp does not match column 'label' type VARCHAR", + 7, + 41, + ), + ], + }, + ModelMetadataFindings::default(), + ], + sources: vec![( + 0, + metadata_finding( + "B300", + "cursor_column references unknown column 'loaded_at'", + 3, + 20, + ), + )], + sql_tests: vec![( + 0, + metadata_finding( + "B302", + "SQL test '__ref__stg' names unknown column 'refund'", + 2, + 48, + ), + 54, + )], + fallback_types: Vec::new(), + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_argument_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_argument_types.rs new file mode 100644 index 0000000000..60d5ddfa73 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_argument_types.rs @@ -0,0 +1,71 @@ +use crate::semantic_checks::tests::helpers::projected_type; +use crate::semantic_checks::tests::test_types::ArgumentTypeTestCase; + +#[test] +fn given_argument_expressions_when_typing_then_matches_python_expression_types() { + let test_cases = [ + ArgumentTypeTestCase { + description: "a sized VARCHAR cast", + expression: "CAST(a AS VARCHAR(10))", + expected_type: Some("VARCHAR(10)"), + }, + ArgumentTypeTestCase { + description: "a DECIMAL cast with precision and scale", + expression: "CAST(a AS DECIMAL(10, 2))", + expected_type: Some("DECIMAL(10, 2)"), + }, + ArgumentTypeTestCase { + description: "a DECIMAL cast with precision only", + expression: "CAST(a AS DECIMAL(10))", + expected_type: Some("DECIMAL(10)"), + }, + ArgumentTypeTestCase { + description: "a TRY_CAST to a zoned timestamp", + expression: "TRY_CAST(a AS TIMESTAMPTZ)", + expected_type: Some("TIMESTAMPTZ"), + }, + ArgumentTypeTestCase { + description: "an unzoned timestamp cast", + expression: "CAST(a AS TIMESTAMP)", + expected_type: Some("TIMESTAMP"), + }, + ArgumentTypeTestCase { + description: "a BIGINT cast", + expression: "CAST(a AS BIGINT)", + expected_type: Some("BIGINT"), + }, + ArgumentTypeTestCase { + description: "a custom type is upper-cased", + expression: "CAST(a AS my_type)", + expected_type: Some("MY_TYPE"), + }, + ArgumentTypeTestCase { + description: "an unmapped serialised name is spaced and upper-cased", + expression: "CAST(a AS HUGEINT)", + expected_type: Some("INT128"), + }, + ArgumentTypeTestCase { + description: "a comparison is Boolean", + expression: "a = 1", + expected_type: Some("BOOLEAN"), + }, + ArgumentTypeTestCase { + description: "a declared function return type", + expression: "my_fn(a)", + expected_type: Some("DATE"), + }, + ArgumentTypeTestCase { + description: "arithmetic has no inferred type", + expression: "a + 1", + expected_type: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + projected_type(test_case.expression).as_deref(), + test_case.expected_type, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs index ae5aecd7ca..c1bb607569 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs @@ -1,4 +1,4 @@ -use crate::semantic_checks::_helpers::columns::{closest_column, ordered_columns}; +use crate::semantic_checks::_helpers::explanation::columns::{closest_column, ordered_columns}; use crate::semantic_checks::models::SemanticDeferral; use crate::semantic_checks::tests::helpers::shape; use crate::semantic_checks::tests::test_types::ClosestColumnTestCase; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs index b0769a0568..b3b665e148 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs @@ -1,4 +1,4 @@ -use crate::semantic_checks::_helpers::messages::comparison_help; +use crate::semantic_checks::_helpers::explanation::messages::comparison_help; use crate::semantic_checks::models::SemanticDeferral; use crate::semantic_checks::tests::helpers::{missing_parts, owned_missing, sentence}; use crate::semantic_checks::tests::test_types::{ComparisonHelpTestCase, SentenceMessageTestCase}; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_metadata.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_metadata.rs new file mode 100644 index 0000000000..cf0c72bd2b --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_metadata.rs @@ -0,0 +1,41 @@ +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::tests::helpers::{expected_checked_outcome, metadata_outcome}; +use crate::semantic_checks::tests::test_types::MetadataTestCase; + +const MART_SHAPE: &[(&str, &str)] = &[ + ("order_id", "INTEGER"), + ("scaled", "DOUBLE"), + ("label", "VARCHAR"), +]; + +#[test] +fn given_project_metadata_when_checking_then_reports_python_errors_or_defers() { + let test_cases = [ + MetadataTestCase { + description: "function arity and families, config references, cursors and SQL tests", + dialect: "duckdb", + mart_shape: MART_SHAPE, + expected_outcome: Ok(expected_checked_outcome()), + }, + MetadataTestCase { + description: "a dialect outside the native parser build", + dialect: "mysql", + mart_shape: MART_SHAPE, + expected_outcome: Err(SemanticDeferral::UnsupportedDialect), + }, + MetadataTestCase { + description: "a column name Python case-folds beyond ASCII", + dialect: "duckdb", + mart_shape: &[("stra\u{df}e", "VARCHAR")], + expected_outcome: Err(SemanticDeferral::NonAsciiText), + }, + ]; + for test_case in test_cases { + assert_eq!( + metadata_outcome(&test_case), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs index e09cd235ce..fa69b6ac56 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs @@ -1,4 +1,4 @@ -use crate::semantic_checks::_helpers::parsed_sql::projection_spans; +use crate::semantic_checks::_helpers::sql_text::parsed_sql::projection_spans; use crate::semantic_checks::models::SemanticDeferral; use crate::semantic_checks::tests::helpers::{names, pairs, parsed_facts}; use crate::semantic_checks::tests::test_types::{ParsedFactsTestCase, ProjectionSpanTestCase}; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_positions.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_positions.rs new file mode 100644 index 0000000000..bc56c3c127 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_positions.rs @@ -0,0 +1,66 @@ +use crate::semantic_checks::_helpers::metadata_checks::positions::text_position; +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::tests::test_types::TextPositionTestCase; + +#[test] +fn given_names_in_text_when_locating_then_matches_python_whole_word_positions() { + let test_cases = [ + TextPositionTestCase { + description: "a longer word sharing the prefix is skipped", + text: "SELECT amount_total, amount\nFROM t", + name: "amount", + offset: 0, + expected_position: Ok((1, 22)), + }, + TextPositionTestCase { + description: "a later line", + text: "x\ny amount", + name: "amount", + offset: 0, + expected_position: Ok((2, 3)), + }, + TextPositionTestCase { + description: "an absent name points at the offset", + text: "abc", + name: "zzz", + offset: 0, + expected_position: Ok((1, 1)), + }, + TextPositionTestCase { + description: "the search starts at the offset", + text: "a amount\n-- amount", + name: "amount", + offset: 3, + expected_position: Ok((2, 4)), + }, + TextPositionTestCase { + description: "columns count code points after non-ASCII text", + text: "caf\u{e9} amount", + name: "amount", + offset: 0, + expected_position: Ok((1, 6)), + }, + TextPositionTestCase { + description: "an empty name in empty text", + text: "", + name: "", + offset: 0, + expected_position: Ok((1, 1)), + }, + TextPositionTestCase { + description: "a non-ASCII neighbour Python's word class may read differently", + text: "\u{e9}amount", + name: "amount", + offset: 0, + expected_position: Err(SemanticDeferral::NonAsciiText), + }, + ]; + for test_case in test_cases { + assert_eq!( + text_position(test_case.text, test_case.name, test_case.offset), + test_case.expected_position, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs index b8f0a329c8..4645a0cf36 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::models::{MetadataOutcome, SemanticDeferral}; /// One located `(code, message, help, notes, location)` diagnostic of a completed project. pub(crate) type DescribedDiagnostic = ( @@ -73,3 +73,24 @@ pub(crate) struct OperandTypeTestCase { pub(crate) shapes: &'static [(&'static str, &'static [(&'static str, &'static str)])], pub(crate) expected_diagnostics: Vec, } + +pub(crate) struct TextPositionTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) name: &'static str, + pub(crate) offset: usize, + pub(crate) expected_position: Result<(i64, i64), SemanticDeferral>, +} + +pub(crate) struct ArgumentTypeTestCase { + pub(crate) description: &'static str, + pub(crate) expression: &'static str, + pub(crate) expected_type: Option<&'static str>, +} + +pub(crate) struct MetadataTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) mart_shape: &'static [(&'static str, &'static str)], + pub(crate) expected_outcome: Result, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs new file mode 100644 index 0000000000..823a04998c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -0,0 +1,155 @@ +//! Native semantic metadata checks: config references, cursors, function calls and SQL tests. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; +use sqlbuild_analysis::semantic_checks::models::{ + MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, MetadataSource, + MetadataSqlTest, ModelMetadataFindings, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +/// `(key, names)` pairs in Python's order. +type NamesInput = Vec<(String, Vec)>; +/// `(name, query, authored, checked, calls_functions, references, cursor_inputs, cursor, type)`. +type ModelInput = ( + String, + String, + String, + bool, + bool, + NamesInput, + NamesInput, + Option, + Option, +); +/// `(dialect, return types, functions, shapes, models, sources, SQL tests)`. +type RequestInput = ( + Option, + Vec<(String, String)>, + Vec<(String, Vec<(String, String)>)>, + Vec<(String, Vec<(String, String)>)>, + Vec, + Vec<(String, Option, String)>, + Vec<(String, NamesInput)>, +); +/// `(code, message, line, column)`. +type ErrorRow = (&'static str, String, i64, i64); +/// `(deferral, [(function, reference findings)], source, SQL test findings, fallback types)`. +type OutcomeRow = ( + Option<&'static str>, + Vec<(Vec, Vec)>, + Vec<(usize, ErrorRow)>, + Vec<(usize, ErrorRow, i64)>, + Vec, +); + +/// Run the metadata checks on `catalog`'s analysis pool. +#[pyfunction] +fn check_semantic_metadata_rows( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + request: RequestInput, +) -> PyResult { + let request = metadata_request(request); + let catalog = &catalog.inner; + let checked = py + .compiler_detach(|| check_semantic_metadata(&request, catalog)) + .map_err(compiler_error)?; + Ok(match checked { + Err(deferral) => ( + Some(deferral.as_str()), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + ), + Ok(outcome) => ( + None, + outcome.models.into_iter().map(model_rows).collect(), + outcome + .sources + .into_iter() + .map(|(index, error)| (index, error_row(error))) + .collect(), + outcome + .sql_tests + .into_iter() + .map(|(index, error, end)| (index, error_row(error), end)) + .collect(), + outcome.fallback_types, + ), + }) +} + +fn error_row(error: MetadataFinding) -> ErrorRow { + (error.code, error.message, error.line, error.column) +} + +fn model_rows(errors: ModelMetadataFindings) -> (Vec, Vec) { + ( + errors.functions.into_iter().map(error_row).collect(), + errors.references.into_iter().map(error_row).collect(), + ) +} + +fn metadata_request( + (dialect, return_types, functions, shapes, models, sources, sql_tests): RequestInput, +) -> MetadataRequest { + MetadataRequest { + dialect, + return_types, + functions: functions + .into_iter() + .map(|(key, arguments)| MetadataFunction { key, arguments }) + .collect(), + shapes, + models: models.into_iter().map(metadata_model).collect(), + sources: sources + .into_iter() + .map(|(name, cursor_column, contents)| MetadataSource { + name, + cursor_column, + contents, + }) + .collect(), + sql_tests: sql_tests + .into_iter() + .map(|(contents, ctes)| MetadataSqlTest { contents, ctes }) + .collect(), + } +} + +fn metadata_model( + ( + name, + query_sql, + authored_sql, + checked, + calls_functions, + references, + cursor_inputs, + cursor, + cursor_type, + ): ModelInput, +) -> MetadataModel { + MetadataModel { + name, + query_sql, + authored_sql, + checked, + calls_functions, + references, + cursor_inputs, + cursor, + cursor_type, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(check_semantic_metadata_rows, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 03829ea183..581ba801c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod completion; +pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 1ad08dbada..b4c9ba8dae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_checks::{completion, type_recovery}; +use crate::bindings::_helpers::semantic_checks::{completion, metadata, type_recovery}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { type_recovery::register(module)?; + metadata::register(module)?; completion::register(module) } diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py index 62a3295c68..ddf0404a2c 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -6,7 +6,12 @@ mart_body_files, staging_files, ) -from scripts.compiler_differential.constants import FAILURE_BASE_STAGING, FAILURE_MART_PATH +from scripts.compiler_differential.constants import ( + FAILURE_BASE_SOURCES, + FAILURE_BASE_STAGING, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, +) from scripts.compiler_differential.models import FailureCase _ORDERS: str = '__ref("stg_orders")' @@ -14,6 +19,25 @@ "amount, status", "amount, status, CAST('2026-01-01' AS TIMESTAMP) AS ordered_at" ) +_FUNCTION_PATH: str = "functions/sql/scaled_amount.sql" +_SCALED_AMOUNT: str = ( + "FUNCTION (\n" + ' description "Scale an amount by an integer factor",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n" + " returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" +) +_LOADED_SOURCE: str = ( + " - name: raw_events\n description: Loaded events.\n managed: true\n" + " write_strategy: append\n contract: enforced\n cursor_column: loaded_at\n" + " columns:\n - name: event_id\n type: INTEGER\n" + " - name: load_seq\n type: INTEGER\n" +) +_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_events(ctx):\n '''Load events.'''\n return [{'event_id': 1, 'load_seq': 1}]\n" +) + def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" @@ -92,4 +116,52 @@ def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: ), }, ), + *_metadata_cases(), + ) + + +def _metadata_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="semantic-udf-qualified-cast-argument-family", + expected_code="B301", + expected_message=( + "Function 'scaled_amount' argument 'p_factor' expects INTEGER, received TEXT" + ), + files={ + _FUNCTION_PATH: _SCALED_AMOUNT, + **mart_body_files( + "SELECT o.customer_id,\n" + ' __udf("scaled_amount")(o.amount, CAST(o.status AS VARCHAR))' + " AS total_amount\n" + f"FROM {_ORDERS} AS o\n" + ), + }, + ), + failure_case( + name="semantic-cursor-inputs-unknown-column", + expected_code="B300", + expected_message="cursor_inputs stg_orders references unknown column 'placed_at'", + files={ + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " unique_key [customer_id],\n cursor customer_id,\n" + " cursor_type integer,\n" + " cursor_inputs (\n stg_orders placed_at,\n ),\n);\n\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ) + }, + ), + failure_case( + name="semantic-source-cursor-unknown-column", + expected_code="B300", + expected_message="cursor_column references unknown column 'loaded_at'", + expected_location=(21, 20), + files={ + FAILURE_SOURCES_PATH: FAILURE_BASE_SOURCES + _LOADED_SOURCE, + "python/loaders/raw_events.py": _LOADER, + }, + ), ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1cd39de451..9949085ec4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -443,6 +443,37 @@ def plan_semantic_type_recovery( ], /, ) -> SemanticTypeRecovery: ... +def check_semantic_metadata_rows( + catalog: object, + request: tuple[ + str | None, + list[tuple[str, str]], + list[tuple[str, list[tuple[str, str]]]], + list[tuple[str, list[tuple[str, str]]]], + list[ + tuple[ + str, + str, + str, + bool, + bool, + list[tuple[str, list[str]]], + list[tuple[str, list[str]]], + str | None, + str | None, + ] + ], + list[tuple[str, str | None, str]], + list[tuple[str, list[tuple[str, list[str]]]]], + ], + /, +) -> tuple[ + str | None, + list[tuple[list[tuple[str, str, int, int]], list[tuple[str, str, int, int]]]], + list[tuple[int, tuple[str, str, int, int]]], + list[tuple[int, tuple[str, str, int, int], int]], + list[str], +]: ... def complete_semantic_checks( catalog: object, request: tuple[ diff --git a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py index a617ba2114..a4cb6fb255 100644 --- a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py +++ b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py @@ -7,6 +7,9 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.analysis.compact import get_complete_schema_binding_request from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( + _audit_errors, + _names, + _sql_test_columns, get_semantic_metadata_diagnostics, ) from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( @@ -14,6 +17,7 @@ known_function_names, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql import get_resource_sql_diagnostics from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest @@ -64,3 +68,51 @@ def metadata_diagnostics( return get_semantic_metadata_diagnostics( project=project, profile=profile, resource_sql_analysis=resource_sql_analysis ) + + @staticmethod + def metadata_shapes( + *, project: CompiledProject, profile: ExpressionInferenceProfile + ) -> dict[str, dict[str, str]]: + """Closed relation shapes, as the metadata checks read them under `profile`.""" + + return semantic_shapes(project=project, profile=profile) + + @staticmethod + def column_names(value: object) -> tuple[str, ...]: + """The column names a metadata config value lists.""" + + return _names(value) + + @staticmethod + def audit_diagnostics( + *, + model: CompiledModel, + project: CompiledProject, + shape: dict[str, str], + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, + ) -> tuple[CompilerDiagnostic, ...]: + """The schema audit checks of one model with a closed shape.""" + + return _audit_errors( + model=model, project=project, shape=shape, shapes=shapes, profile=profile + ) + + @staticmethod + def sql_test_columns( + *, project: CompiledProject, profile: ExpressionInferenceProfile + ) -> dict[str, tuple[str, ...]]: + """The inferred output columns of every model SQL test CTE body.""" + + return _sql_test_columns(project=project, profile=profile) + + @staticmethod + def resource_sql_diagnostics( + *, + project: CompiledProject, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, + ) -> tuple[CompilerDiagnostic, ...]: + """The compile-time SQL checks of audits, SQL tests and SQL hooks.""" + + return tuple(get_resource_sql_diagnostics(project=project, shapes=shapes, profile=profile)) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py new file mode 100644 index 0000000000..3c1db60355 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py @@ -0,0 +1,235 @@ +"""Native semantic metadata checks, with Python's audit and resource SQL checks in order.""" + +from __future__ import annotations + +from typing import Any, cast + +import sqlbuild._native as _native +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapter.type_system.main.normalize_type import normalize_type +from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs +from sqlbuild.compiler.compile.models import ( + CompiledFunction, + CompiledModel, + CompiledModelSqlTestPayload, + CompiledProject, + CompiledSource, + CompiledSqlTest, + CompilerDiagnostic, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY +from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral +from sqlbuild.compiler.semantic_checks.constants import ( + METADATA_CONFIG_KEYS, + METADATA_DEFERRAL_SITE, + METADATA_FUNCTION_REFERENCE_KINDS, +) +from sqlbuild.spec.contracts.models import SourceLocation + +type _ErrorRow = tuple[str, str, int, int] +type _NamesRow = tuple[str, list[str]] + + +def native_metadata_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + resource_sql_analysis: bool, + catalog: object, +) -> tuple[CompilerDiagnostic, ...] | None: + """Return `get_semantic_metadata_diagnostics`' result, or None where Python must check.""" + + if not project.settings.sql_analysis: + return () + shapes: dict[str, dict[str, str]] = SemanticCompletionInputs.metadata_shapes( + project=project, profile=profile + ) + tests: list[CompiledSqlTest] = [ + test for test in project.sql_tests if isinstance(test.payload, CompiledModelSqlTestPayload) + ] + columns_by_sql: dict[str, tuple[str, ...]] = SemanticCompletionInputs.sql_test_columns( + project=project, profile=profile + ) + functions: dict[str, CompiledFunction] = { + function.name.casefold(): function for function in project.functions + } + request: Any = ( + profile.sql_analysis_dialect, + list(profile.function_return_types.items()), + [_function_payload(key=key, function=function) for key, function in functions.items()], + [(name, list(columns.items())) for name, columns in shapes.items()], + [_model_payload(model) for model in project.models], + [_source_payload(source) for source in project.sources], + [_sql_test_payload(test=test, columns_by_sql=columns_by_sql) for test in tests], + ) + deferral, model_rows, source_rows, test_rows, fallback_types = ( + _native.check_semantic_metadata_rows(catalog, request) + ) + if deferral is not None: + record_semantic_deferral(kind=deferral, site=METADATA_DEFERRAL_SITE) + return None + for type_sql in fallback_types: + _ = normalize_type(type_sql=type_sql, dialect=profile.sql_analysis_dialect) + diagnostics: list[CompilerDiagnostic] = [] + for model, (function_rows, reference_rows) in zip(project.models, model_rows, strict=True): + if not _checked(model): + continue + diagnostics.extend(_model_error(model=model, row=row) for row in function_rows) + shape: dict[str, str] | None = shapes.get(model.name) + if shape is None: + continue + diagnostics.extend( + SemanticCompletionInputs.audit_diagnostics( + model=model, project=project, shape=shape, shapes=shapes, profile=profile + ) + ) + diagnostics.extend(_model_error(model=model, row=row) for row in reference_rows) + diagnostics.extend( + _source_error(source=project.sources[index], row=row) for index, row in source_rows + ) + diagnostics.extend( + _sql_test_error(test=tests[index], row=row, end_column=end_column) + for index, row, end_column in test_rows + ) + if resource_sql_analysis: + diagnostics.extend( + SemanticCompletionInputs.resource_sql_diagnostics( + project=project, shapes=shapes, profile=profile + ) + ) + return tuple(diagnostics) + + +def _function_payload(*, key: str, function: CompiledFunction) -> tuple[str, list[tuple[str, str]]]: + return (key, [(argument.name, argument.type) for argument in function.arguments]) + + +def _checked(model: CompiledModel) -> bool: + return model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is not False and model.binding_validated + + +def _string(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _model_payload(model: CompiledModel) -> tuple[Any, ...]: + values: dict[str, object] = model.config.values + references: list[_NamesRow] = [ + (key, list(SemanticCompletionInputs.column_names(values.get(key)))) + for key in METADATA_CONFIG_KEYS + ] + custom: object = values.get("config") + if isinstance(custom, dict): + partition: object = cast(dict[str, object], custom).get("partition_column") + references.append( + ("partition_column", list(SemanticCompletionInputs.column_names(partition))) + ) + tolerances: object = values.get("row_diff_tolerances") + by_column: object = ( + cast(dict[str, object], tolerances).get("by_column") + if isinstance(tolerances, dict) + else None + ) + if isinstance(by_column, dict): + references.append(("row_diff_tolerances", [str(name) for name in by_column])) + cursor_inputs: object = values.get("cursor_inputs") + upstreams: list[_NamesRow] = [] + if isinstance(cursor_inputs, dict): + upstreams = [ + (str(upstream), list(SemanticCompletionInputs.column_names(columns))) + for upstream, columns in cursor_inputs.items() + ] + return ( + model.name, + model.query_sql, + model.authored_sql, + _checked(model), + any( + reference.ref_kind in METADATA_FUNCTION_REFERENCE_KINDS + for reference in model.references + ), + references, + upstreams, + _string(values.get("cursor")), + _string(values.get("cursor_type")), + ) + + +def _source_payload(source: CompiledSource) -> tuple[str, str | None, str]: + return ( + source.name, + source.source_entry.cursor_column or None, + source.source_file.contents, + ) + + +def _sql_test_payload( + *, test: CompiledSqlTest, columns_by_sql: dict[str, tuple[str, ...]] +) -> tuple[str, list[_NamesRow]]: + payload: CompiledModelSqlTestPayload = cast(CompiledModelSqlTestPayload, test.payload) + return ( + test.test_file.contents, + [ + (cte.name, list(columns_by_sql[cte.sql_body])) + for cte in (*payload.authored_ctes, *payload.expected_ctes) + ], + ) + + +def _model_error(*, model: CompiledModel, row: _ErrorRow) -> CompilerDiagnostic: + code, message, line, column = row + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=code, + message=message, + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + path=model.relative_path, + line=line, + column=column, + ) + + +def _source_error(*, source: CompiledSource, row: _ErrorRow) -> CompilerDiagnostic: + code, message, line, column = row + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=code, + message=message, + resource_type=CompiledResourceType.SOURCE, + resource_name=source.name, + path=source.source_file.relative_path, + line=line, + column=column, + ) + + +def _sql_test_error( + *, test: CompiledSqlTest, row: _ErrorRow, end_column: int +) -> CompilerDiagnostic: + code, message, line, column = row + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=code, + message=message, + resource_type=CompiledResourceType.SQL_TEST, + resource_name=test.name, + path=test.test_file.relative_path, + line=line, + column=column, + location=SourceLocation( + path=test.test_file.relative_path, + line=line, + column=column, + end_line=line, + end_column=end_column, + ), + ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py index 46e18968c8..1468d8c319 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -9,11 +9,10 @@ from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral +from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types from sqlbuild.compiler.semantic_checks.constants import ( COMPLETION_DEFERRAL_SITE, - METADATA_DEFERRAL_SITE, - NATIVE_SEMANTIC_METADATA_CHECKS, NATIVE_SEMANTIC_NO_CATALOG, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic @@ -37,11 +36,16 @@ def completed_semantic_project( ) if recovered is None: return None - if recovered.settings.sql_analysis: - record_semantic_deferral(kind=NATIVE_SEMANTIC_METADATA_CHECKS, site=METADATA_DEFERRAL_SITE) - metadata: tuple[CompilerDiagnostic, ...] = SemanticCompletionInputs.metadata_diagnostics( - project=recovered, profile=profile, resource_sql_analysis=resource_sql_analysis + metadata: tuple[CompilerDiagnostic, ...] | None = native_metadata_diagnostics( + project=recovered, + profile=profile, + resource_sql_analysis=resource_sql_analysis, + catalog=catalog, ) + if metadata is None: + metadata = SemanticCompletionInputs.metadata_diagnostics( + project=recovered, profile=profile, resource_sql_analysis=resource_sql_analysis + ) return complete_native_diagnostics( project=replace(recovered, diagnostics=(*recovered.diagnostics, *metadata)), catalog=catalog, diff --git a/src/sqlbuild/compiler/semantic_checks/constants.py b/src/sqlbuild/compiler/semantic_checks/constants.py index 6ea316eb6f..53d2036c0e 100644 --- a/src/sqlbuild/compiler/semantic_checks/constants.py +++ b/src/sqlbuild/compiler/semantic_checks/constants.py @@ -1,11 +1,21 @@ """Native semantic completion statuses and deferral record sites.""" +from sqlbuild.compiler.references.types import SqlReferenceKind + NATIVE_TYPE_RECOVERY_UNCHANGED: str = "unchanged" NATIVE_TYPE_RECOVERY_DEFERRED: str = "deferred" NATIVE_SEMANTIC_NO_CATALOG: str = "no_analysis_catalog" -NATIVE_SEMANTIC_METADATA_CHECKS: str = "metadata_checks" NATIVE_SEMANTIC_FAILURE: str = "native_failure" TYPE_RECOVERY_DEFERRAL_SITE: str = "type_recovery.py" COMPLETION_DEFERRAL_SITE: str = "recovery.py" METADATA_DEFERRAL_SITE: str = "metadata_validation.py" UNKNOWN_RECOVERED_TYPE: str = "UNKNOWN" +METADATA_CONFIG_KEYS: tuple[str, ...] = ( + "unique_key", + "cursor", + "partition_column", + "row_diff_exclude_columns", +) +METADATA_FUNCTION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( + {SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION} +) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index f0184ff6c8..62806b8a70 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -443,7 +443,6 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness "Polyglot wheel calls (native-preview):", "Analysis deferrals (python): none recorded", "Analysis deferrals (native-preview):", - " metadata_checks metadata_validation.py (project ", " input_enrichment analysis_session (project ", "Compiler differential passed: 1 projects identical", ), @@ -453,9 +452,7 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness "_validate_sql_syntax_with_message parse_one", } ), - expected_deferrals=frozenset( - {"metadata_checks metadata_validation.py", "input_enrichment analysis_session"} - ), + expected_deferrals=frozenset({"input_enrichment analysis_session"}), ), WheelSiteReportTestCase( description="preview_deferrals_are_counted_per_kind_and_site", @@ -469,7 +466,6 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness expected_deferrals=frozenset( { "legacy_fallback orders.sql", - "metadata_checks metadata_validation.py", "input_enrichment analysis_session", } ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py index 59b7847e73..a1b15992d4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -19,6 +19,7 @@ SEMANTIC_WHEEL_SITES: tuple[str, ...] = ( "compiler/compile/_helpers/diagnostics/type_recovery.py:_projection_spans", "compiler/compile/_helpers/diagnostics/details.py:_parsed_model", + "compiler/compile/_helpers/assembly/metadata_validation.py:_function_errors", ) SEMANTIC_DEFERRAL_SITES: frozenset[str] = frozenset( {COMPLETION_DEFERRAL_SITE, METADATA_DEFERRAL_SITE, TYPE_RECOVERY_DEFERRAL_SITE} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks_cli.py similarity index 61% rename from tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py rename to tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks_cli.py index 68d5eb9047..259c102e38 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks_cli.py @@ -47,6 +47,47 @@ 'FROM __ref("stg_orders")\n' ), } +_METADATA_FILES: dict[str, str] = { + "sqlbuild_project.toml": _CONFIG, + "sources/raw.yml": _SOURCES + + ( + " - name: raw_events\n description: Loaded events.\n managed: true\n" + " write_strategy: append\n contract: enforced\n cursor_column: loaded_at\n" + " columns:\n - name: event_id\n type: INTEGER\n" + " - name: load_seq\n type: INTEGER\n" + ), + "python/loaders/raw_events.py": ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_events(ctx):\n '''Load events.'''\n" + " return [{'event_id': 1, 'load_seq': 1}]\n" + ), + "functions/sql/scaled_amount.sql": ( + 'FUNCTION (\n description "Scale an amount",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" + ), + "models/staging/stg_orders.sql": ( + 'MODEL (description "Staged orders");\n\n' + "SELECT order_id, customer_id, amount, status, ordered_at\n" + 'FROM __source("raw_orders")\n' + ), + "models/marts/customer_totals.sql": ( + 'MODEL (\n description "Totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " unique_key [customer_key],\n cursor ordered_at,\n cursor_type integer,\n" + " cursor_inputs (\n stg_orders placed_at,\n ),\n);\n\n" + "SELECT o.customer_id, MAX(o.ordered_at) AS ordered_at,\n" + ' SUM(__udf("scaled_amount")(o.status, 2)) AS scaled_total,\n' + ' MAX(__udf("scaled_amount")(o.amount)) AS top_amount\n' + 'FROM __ref("stg_orders") AS o\nGROUP BY o.customer_id\n' + ), + "tests/unit/test_stg_orders.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status, TIMESTAMP '2026-01-01' AS ordered_at\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id, 2 AS discount\n)\nSELECT 1\n" + ), +} @pytest.mark.parametrize( @@ -62,7 +103,14 @@ "o.ordered_at is TIMESTAMP, 5 is INTEGER", "raw_orders has: amount, status, ordered_at, customer_id, order_id", ), - expected_preview_deferrals=(("metadata_checks", "metadata_validation.py"),), + expected_preview_deferrals=(), + ), + NativeSemanticChecksCliTestCase( + description="UDF calls, config and cursor references, a source cursor and a SQL test", + files=_METADATA_FILES, + expected_codes=("B301", "B102", "B300", "B300", "B301", "B300", "B302"), + expected_notes=(), + expected_preview_deferrals=(), ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py index e3f9874dc0..31dd072d29 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py @@ -27,3 +27,16 @@ class DeferredSemanticTestCase: keeps_catalog: bool non_ascii_comment: bool expected_kinds: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class GeneratedMetadataParityTestCase: + """Seeded projects whose metadata checks Python and the native engine must agree on.""" + + description: str + seed: int + count: int + model_count: int + dialects: tuple[str, ...] + expected_minimum_native: int + expected_minimum_families: dict[str, int] diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 112162a298..97ed5ccb0d 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -7,7 +7,7 @@ from dataclasses import dataclass, replace from itertools import chain from pathlib import Path -from typing import Any +from typing import Any, cast import pytest @@ -15,13 +15,17 @@ import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( + get_semantic_metadata_diagnostics, +) from sqlbuild.compiler.compile._helpers.diagnostics.recovery import ( complete_semantic_diagnostics, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( complete_native_semantic_diagnostics, ) @@ -297,3 +301,206 @@ def deferral_records(record_dir: Path) -> tuple[tuple[str, str], ...]: record: dict[str, str] = json.loads(line) records.append((record["kind"], record["site"])) return tuple(records) + + +_SOURCE_COLUMNS: tuple[str, ...] = ( + "order_id", + "customer_id", + "amount", + "status", + "ordered_at", + "order_date", +) +_MODEL_COLUMNS: tuple[str, ...] = ("order_id", "customer_id", "ordered_at", "value") +_FUNCTIONS: dict[str, tuple[str, str, str, int]] = { + "scaled_amount": ("p_amount DOUBLE, p_factor INTEGER", "DOUBLE", "p_amount * p_factor", 2), + "label_status": ("p_status VARCHAR", "VARCHAR", "UPPER(p_status)", 1), + "days_since": ("p_at TIMESTAMP", "INTEGER", "DATE_DIFF('day', p_at, NOW())", 1), +} +_ARGUMENTS: tuple[str, ...] = ( + "o.{column}", + "{column}", + "x.{column}", + "'x'", + "2", + "2.5", + "CAST({column} AS VARCHAR)", + "CAST({column} AS DECIMAL(10, 2))", + "TRUE", + "{column} = 1", +) +_ARITY_CHANGES: tuple[int, ...] = (0, 0, 0, 1, -1) +_UNIQUE_KEYS: tuple[str, ...] = ("order_id", "customer_id", "order_key") +_CURSORS: tuple[str, ...] = ("ordered_at", "customer_id", "value", "placed_at") +_CURSOR_TYPES: tuple[str, ...] = ("timestamp", "integer") +_GRAINS: dict[str, str] = {"timestamp": " cursor_grain day,\n"} +_INPUT_COLUMNS: tuple[str, ...] = ("ordered_at", "order_id", "created_at") +_LOADED_CURSORS: tuple[str, ...] = ("load_seq", "Load_Seq", "loaded_at") +_LOADED_SOURCE: str = ( + " - name: raw_events\n description: Loaded events.\n managed: true\n" + " write_strategy: append\n contract: enforced\n cursor_column: {cursor}\n" + " columns:\n" + " - name: event_id\n type: INTEGER\n" + " - name: load_seq\n type: INTEGER\n" +) +_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n\n@loader\n" + "def raw_events(ctx):\n '''Load events.'''\n return [{'event_id': 1, 'load_seq': 1}]\n" +) +_FIXTURE_EXTRAS: tuple[str, ...] = ("", ", 2 AS discount") +_EXPECTED_EXTRAS: tuple[str, ...] = ("", ", 3 AS Customer_ID", ", 2 AS refund") +_CTE_SHARE: float = 0.3 +_AUDITS: tuple[str, ...] = ( + "", + " columns (\n customer_id (audits [accepted_values (values [DATE '2026-01-01'])]),\n ),\n", + ' columns (\n order_id (audits [relationships (to __ref("orders_model_0"),' + " field order_key)]),\n ),\n", +) +_SOURCE_SQL: str = ( + " expression: >-\n" + " (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS" + " status,\n TIMESTAMP '2026-01-01 00:00:00' AS ordered_at, DATE '2026-01-01' AS" + " order_date)\n columns:\n" +) +_SOURCE_TYPES: tuple[str, ...] = ("INTEGER", "INTEGER", "DOUBLE", "VARCHAR", "TIMESTAMP", "DATE") + +type _Relation = tuple[str, str, tuple[str, ...]] + + +def _function_file(name: str) -> str: + arguments, returns, body, _ = _FUNCTIONS[name] + return ( + f'FUNCTION (\n description "Generated {name}",\n arguments ({arguments}),\n' + f" returns {returns},\n);\n\n{body}\n" + ) + + +def _call(rng: random.Random, columns: tuple[str, ...], alias: str) -> str: + name: str = rng.choice(tuple(_FUNCTIONS)) + count: int = max(0, _FUNCTIONS[name][3] + rng.choice(_ARITY_CHANGES)) + arguments: list[str] = [ + rng.choice(_ARGUMENTS).format(column=rng.choice(columns)).replace("o.", f"{alias}.") + for _ in range(count) + ] + return f'__udf("{name}")({", ".join(arguments)})' + + +def _query(rng: random.Random, relation: _Relation) -> str: + reference, _, columns = relation + plain: str = ( + "SELECT o.order_id, o.customer_id, o.ordered_at, " + f"{_call(rng, columns, 'o')} AS value\nFROM {reference} AS o" + ) + with_cte: str = ( + f"WITH base AS (\n SELECT * FROM {reference} AS o\n)\n" + f"SELECT b.order_id, b.customer_id, b.ordered_at, {_call(rng, columns, 'b')} AS value\n" + "FROM base AS b" + ) + return (plain, with_cte)[rng.random() < _CTE_SHARE] + + +def _header(rng: random.Random, index: int, relation: _Relation) -> str: + _, upstream, _ = relation + cursor_type: str = rng.choice(_CURSOR_TYPES) + incremental: str = ( + " materialized incremental,\n incremental_strategy delete_insert,\n" + f" unique_key [order_id],\n cursor {rng.choice(_CURSORS)},\n" + f" cursor_type {cursor_type},\n{_GRAINS.get(cursor_type, '')}" + f" cursor_inputs (\n {upstream} {rng.choice(_INPUT_COLUMNS)},\n ),\n" + ) + table: str = f" materialized table,\n unique_key [{rng.choice(_UNIQUE_KEYS)}],\n" + configs: tuple[str, str, str] = (table, incremental, "") + kind: int = (index % 3, 0)[index % 3 == 1 and not upstream] + return configs[kind] + rng.choice(_AUDITS) + + +def generated_metadata_files(*, rng: random.Random, model_count: int) -> dict[str, str]: + """A project calling declared functions, with config, source cursor and SQL test metadata.""" + + columns: str = "".join( + f" - name: {name}\n type: {kind}\n" + for name, kind in zip(_SOURCE_COLUMNS, _SOURCE_TYPES, strict=True) + ) + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_metadata"\nadapter = "duckdb"\n\n[connection]\n' + 'database = "orders.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Orders feed.\n" + f"{_SOURCE_SQL}{columns}{_LOADED_SOURCE.format(cursor=rng.choice(_LOADED_CURSORS))}" + ), + "python/loaders/raw_events.py": _LOADER, + "tests/unit/test_orders_model_0.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status, TIMESTAMP '2026-01-01' AS ordered_at," + f" DATE '2026-01-01' AS order_date{rng.choice(_FIXTURE_EXTRAS)}\n),\n" + "__expected__orders_model_0 AS (\n" + f" SELECT 1 AS order_id{rng.choice(_EXPECTED_EXTRAS)}\n)\nSELECT 1\n" + ), + } + for name in _FUNCTIONS: + files[f"functions/sql/{name}.sql"] = _function_file(name) + relations: list[_Relation] = [('__source("raw_orders")', "", _SOURCE_COLUMNS)] + for index in range(model_count): + relation: _Relation = rng.choice(relations) + name: str = f"orders_model_{index}" + files[f"models/layer_{index % 3}/{name}.sql"] = ( + f'MODEL (\n description "Generated orders model {index}",\n' + f"{_header(rng, index, relation)});\n\n{_query(rng, relation)}\n" + ) + relations.append((f'__ref("{name}")', name, _MODEL_COLUMNS)) + return files + + +def python_metadata( + *, inputs: SemanticInputs, monkeypatch: pytest.MonkeyPatch +) -> tuple[CompilerDiagnostic, ...]: + """The Python engine's metadata diagnostics.""" + + with monkeypatch.context() as patch: + patch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + return get_semantic_metadata_diagnostics( + project=inputs.project, + profile=inputs.profile, + resource_sql_analysis=inputs.resource_sql_analysis, + ) + + +def native_metadata(inputs: SemanticInputs) -> tuple[CompilerDiagnostic, ...] | None: + """The native stage's metadata diagnostics, or None when it defers.""" + + catalog: Any = cast(Any, inputs.project.binding_catalog) + return native_metadata_diagnostics( + project=inputs.project, + profile=inputs.profile, + resource_sql_analysis=inputs.resource_sql_analysis, + catalog=catalog.native, + ) + + +def _family_counts(outcome: Any) -> Counter[str]: + counts: Counter[str] = Counter() + for function_rows, reference_rows in outcome[1]: + counts.update(f"function {row[0]}" for row in function_rows) + counts.update(f"reference {row[0]}" for row in reference_rows) + counts.update(f"source {row[0]}" for _, row in outcome[2]) + counts.update(f"sql_test {row[0]}" for _, row, _ in outcome[3]) + return counts + + +def record_metadata_families(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count native metadata runs, deferrals and errors by family and code.""" + + counts: Counter[str] = Counter() + check: Callable[..., Any] = native_module.check_semantic_metadata_rows + + def counted(*arguments: Any) -> Any: + outcome: Any = check(*arguments) + counts[("native", "deferred")[outcome[0] is not None]] += 1 + counts.update(_family_counts(outcome)) + return outcome + + monkeypatch.setattr(native_module, "check_semantic_metadata_rows", counted) + return counts diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py new file mode 100644 index 0000000000..144c7b8c8e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py @@ -0,0 +1,78 @@ +"""Native semantic metadata checks equal Python's on generated projects, family by family.""" + +from __future__ import annotations + +import random +from collections import Counter +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from tests.integration.src.sqlbuild.compiler.semantic_checks._test_types import ( + GeneratedMetadataParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.semantic_checks.helpers import ( + SemanticInputs, + captured_semantic_inputs, + generated_metadata_files, + native_metadata, + python_metadata, + record_metadata_families, + with_dialect, +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedMetadataParityTestCase( + description="function calls, config references, cursors, audits and SQL tests", + seed=20261010, + count=8, + model_count=10, + dialects=("duckdb", "postgres", "snowflake", "bigquery"), + expected_minimum_native=32, + expected_minimum_families={ + "function B102": 100, + "function B301": 100, + "reference B300": 50, + "reference B301": 16, + "source B300": 4, + "sql_test B302": 16, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_projects_when_checking_metadata_natively_then_matches_python( + test_case: GeneratedMetadataParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + families: Counter[str] = record_metadata_families(monkeypatch=monkeypatch) + expected: list[tuple[CompilerDiagnostic, ...] | None] = [] + actual: list[tuple[CompilerDiagnostic, ...] | None] = [] + for index in range(test_case.count): + captured: SemanticInputs = captured_semantic_inputs( + project_dir=tmp_path / f"project_{index}", + files=generated_metadata_files(rng=rng, model_count=test_case.model_count), + monkeypatch=monkeypatch, + ) + for dialect in test_case.dialects: + inputs: SemanticInputs = with_dialect(captured, dialect) + expected.append(python_metadata(inputs=inputs, monkeypatch=monkeypatch)) + actual.append(native_metadata(inputs)) + + assert actual == expected + assert families["deferred"] == 0 + assert families["native"] >= test_case.expected_minimum_native + assert { + family: min(families[family], minimum) + for family, minimum in test_case.expected_minimum_families.items() + } == test_case.expected_minimum_families + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 1aa6c1ab1d..96d98e0bbb 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -110,10 +110,7 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ dialect="duckdb", keeps_catalog=True, non_ascii_comment=True, - expected_kinds=( - ("metadata_checks", "metadata_validation.py"), - ("non_ascii_text", "recovery.py"), - ), + expected_kinds=(("non_ascii_text", "recovery.py"),), ), DeferredSemanticTestCase( description="a project without an analysis catalog to run on", diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py index 04b982d897..232c66c2c5 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -133,3 +133,13 @@ class DefaultEngineStageTestCase: description: str expected_enabled: frozenset[NativeStage] expected_disabled: frozenset[NativeStage] + + +@dataclass(frozen=True) +class NativeStageCouplingTestCase: + """A native stage that may only run where the stage it relies on also runs natively.""" + + description: str + dependent: NativeStage + dependency: NativeStage + expected_engines_without_dependency: frozenset[CompilerEngine] diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index d7ef9561bb..eb3b0d5ce0 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -2,13 +2,20 @@ from __future__ import annotations +from itertools import compress + import pytest -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + ENGINE_NATIVE_STAGE_TIERS, + NATIVE_STAGE_TIERS, +) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import CompilerEngine, NativeStage from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( DefaultEngineStageTestCase, + NativeStageCouplingTestCase, NativeStageTierTestCase, ) @@ -359,5 +366,46 @@ def test_given_no_engine_selection_when_checking_native_stages_then_rendering_ru } +@pytest.mark.parametrize( + "test_case", + [ + NativeStageCouplingTestCase( + description=( + "semantic checks normalize types with the native type system unconditionally, " + "while Python only does so where the type system stage runs natively" + ), + dependent=NativeStage.SEMANTIC_CHECKS, + dependency=NativeStage.TYPE_SYSTEM, + expected_engines_without_dependency=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_coupled_native_stages_when_reading_tiers_then_dependent_never_runs_alone( + test_case: NativeStageCouplingTestCase, +) -> None: + engines: tuple[CompilerEngine, ...] = tuple(ENGINE_NATIVE_STAGE_TIERS) + dependent_engines: frozenset[CompilerEngine] = frozenset( + compress( + engines, + ( + NATIVE_STAGE_TIERS[test_case.dependent] in tiers + for tiers in ENGINE_NATIVE_STAGE_TIERS.values() + ), + ) + ) + dependency_engines: frozenset[CompilerEngine] = frozenset( + compress( + engines, + ( + NATIVE_STAGE_TIERS[test_case.dependency] in tiers + for tiers in ENGINE_NATIVE_STAGE_TIERS.values() + ), + ) + ) + + assert dependent_engines - dependency_engines == (test_case.expected_engines_without_dependency) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])