diff --git a/Makefile b/Makefile index c5e1fd2f08..919fd5ffb8 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \ diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs index 0e356fef98..414cb0b950 100644 --- a/crates/sqlbuild-model-config/src/lib.rs +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -4,6 +4,8 @@ pub mod config_presence; pub mod header_metadata; +pub mod model_validation; +pub mod path_defaults; pub mod templates; pub mod types; diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs new file mode 100644 index 0000000000..e1d386cb32 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs @@ -0,0 +1,130 @@ +//! Effective config values read the way the Python validators read the dict. + +use std::collections::HashMap; + +use crate::model_validation::_helpers::text::{python_lower, python_strip}; +use crate::model_validation::models::Rejected; +use crate::types::{AuthoredNode, NodeKind}; + +/// Config values by string key; Python's `dict.get` never matches other keys. +pub(crate) struct ConfigView { + values: HashMap, +} + +impl ConfigView { + /// View the entries of one config mapping. + pub(crate) fn new(entries: Vec<(N, N)>) -> Self { + let mut values: HashMap = HashMap::with_capacity(entries.len()); + for (key, value) in entries { + if let Some(text) = key.text() { + values.insert(text, value); + } + } + Self { values } + } + + /// Return whether `key in values`, even when its value is `None`. + pub(crate) fn contains_key(&self, key: &str) -> bool { + self.values.contains_key(key) + } + + /// Return `values.get(key)` unless it is absent or `None`. + pub(crate) fn get(&self, key: &str) -> Option<&N> { + self.values + .get(key) + .filter(|value| value.kind() != NodeKind::Null) + } + + /// Return whether `values.get(key) is not None`. + pub(crate) fn present(&self, key: &str) -> bool { + self.get(key).is_some() + } + + /// Return whether any of `keys` is set. + pub(crate) fn any_present(&self, keys: &[&str]) -> bool { + keys.iter().any(|key| self.present(key)) + } + + /// Read `get_config_str`: a string, `None`, or a rejection for any other type. + pub(crate) fn string(&self, key: &str) -> Result, Rejected> { + self.get(key).map(string_value).transpose() + } + + /// Return whether the value equals `text`, as Python's `==` against a string. + pub(crate) fn equals(&self, key: &str, text: &str) -> bool { + self.get(key).is_some_and(|value| value.is_text(text)) + } +} + +/// Return the text of a string value, or reject any other type. +pub(crate) fn string_value(value: &N) -> Result { + if value.kind() == NodeKind::Str { + value.text().ok_or(Rejected) + } else { + Err(Rejected) + } +} + +/// Return `None` for an absent or `None` value, the text of a string, or reject other types. +pub(crate) fn optional_text( + value: Option<&N>, +) -> Result, Rejected> { + match value { + Some(node) if node.kind() != NodeKind::Null => string_value(node).map(Some), + _ => Ok(None), + } +} + +/// Return whether `text in vocabulary`. +pub(crate) fn one_of(text: &str, vocabulary: &[&str]) -> bool { + vocabulary.contains(&text) +} + +/// Return whether an optional string is in `vocabulary`; `None` is not. +pub(crate) fn optional_one_of(text: Option<&str>, vocabulary: &[&str]) -> bool { + text.is_some_and(|value| one_of(value, vocabulary)) +} + +/// Read `_string_sequence`: a string, or the strings of a list or tuple; reject unreadable text. +pub(crate) fn string_sequence(value: Option<&N>) -> Result, Rejected> { + match value.map(|node| (node.kind(), node)) { + Some((NodeKind::Str, node)) => string_value(node).map(|text| vec![text]), + Some((NodeKind::List | NodeKind::Tuple, node)) => node + .items() + .iter() + .filter(|item| item.kind() == NodeKind::Str) + .map(string_value) + .collect(), + _ => Ok(Vec::new()), + } +} + +/// Return whether a value is set and not an empty list or tuple, like `_has_config_value`. +pub(crate) fn has_config_value(value: Option<&N>) -> bool { + value.is_some_and(|node| { + !matches!(node.kind(), NodeKind::List | NodeKind::Tuple) || !node.items().is_empty() + }) +} + +/// Return the text of a string whose Python `strip()` is non-empty, or reject. +pub(crate) fn non_blank_string(value: &N) -> Result { + let text = string_value(value)?; + if python_strip(&text)?.is_empty() { + Err(Rejected) + } else { + Ok(text) + } +} + +/// Reject duplicate names under Python's `str.lower()`. +pub(crate) fn reject_case_insensitive_duplicates(names: &[String]) -> Result<(), Rejected> { + let mut seen = Vec::with_capacity(names.len()); + for name in names { + let lowered = python_lower(name)?; + if seen.contains(&lowered) { + return Err(Rejected); + } + seen.push(lowered); + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs new file mode 100644 index 0000000000..3c10d64100 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs @@ -0,0 +1,184 @@ +//! `cursor_start` and `cursor_end` values Python certainly accepts, with their ordering keys. + +use crate::model_validation::constants::{ + CLOCK_FIELD_LENGTH, CLOCK_PART_COUNTS, CLOCK_PARTS_WITH_SECONDS, INTEGER_CURSOR, + ISO_DATE_LENGTH, ISO_DATE_SEPARATOR_OFFSETS, LAST_SHIFTED_MONTH, MAX_HOUR, MAX_MINUTE, + MAX_YEAR, MIN_YEAR, TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH, +}; +use crate::model_validation::models::Rejected; +use crate::types::AuthoredNode; + +const MICROS_PER_SECOND: i128 = 1_000_000; +const SECONDS_PER_DAY: i128 = 86_400; +const SECONDS_PER_HOUR: i128 = 3_600; +const SECONDS_PER_MINUTE: i128 = 60; +const CIVIL_EPOCH_OFFSET_DAYS: i128 = 719_468; +const FRACTION_DIGITS: [usize; 2] = [3, 6]; + +/// Return the `_cursor_contract_key` order key: UTC microseconds, or the integer itself. +pub(crate) fn cursor_bound_key( + value: &N, + cursor_type: &str, +) -> Result { + if let Some(number) = value.integer().filter(|_| cursor_type == INTEGER_CURSOR) { + return Ok(i128::from(number)); + } + let text = value.text().ok_or(Rejected)?; + match cursor_type { + INTEGER_CURSOR => integer_key(&text), + TIMESTAMP_CURSOR => timestamp_key(&text), + _ => Err(Rejected), + } +} + +/// Return whether two timestamp keys keep their order through Python's float timestamps. +pub(crate) fn keys_ordered(start: i128, end: i128, cursor_type: &str) -> bool { + if cursor_type == TIMESTAMP_CURSOR { + end - start >= MICROS_PER_SECOND + } else { + start < end + } +} + +fn integer_key(text: &str) -> Result { + let digits = text.strip_prefix(['+', '-']).unwrap_or(text); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(Rejected); + } + text.strip_prefix('+') + .unwrap_or(text) + .parse() + .map_err(|_| Rejected) +} + +fn timestamp_key(text: &str) -> Result { + if !text.is_ascii() { + return Err(Rejected); + } + let clock_start = ISO_DATE_LENGTH + 1; + let (date, time) = match text.len() { + ISO_DATE_LENGTH => (text, None), + length + if length > clock_start && matches!(text.as_bytes()[ISO_DATE_LENGTH], b'T' | b' ') => + { + (&text[..ISO_DATE_LENGTH], Some(&text[clock_start..])) + } + _ => return Err(Rejected), + }; + let days = date_days(date)?; + let (micros, offset_seconds) = match time { + None => (0, 0), + Some(clock) => clock_micros(clock)?, + }; + let utc_micros = (days * SECONDS_PER_DAY - offset_seconds) * MICROS_PER_SECOND + micros; + let day_micros = SECONDS_PER_DAY * MICROS_PER_SECOND; + let first_micros = days_from_civil(MIN_YEAR, 1, 1) * day_micros; + let end_micros = days_from_civil(MAX_YEAR + 1, 1, 1) * day_micros; + if utc_micros < first_micros || utc_micros >= end_micros { + return Err(Rejected); + } + Ok(utc_micros) +} + +fn date_days(date: &str) -> Result { + let bytes = date.as_bytes(); + let [year_end, month_end] = ISO_DATE_SEPARATOR_OFFSETS; + if bytes[year_end] != b'-' || bytes[month_end] != b'-' { + return Err(Rejected); + } + let year = number(&date[..year_end])?; + let month = number(&date[year_end + 1..month_end])?; + let day = number(&date[month_end + 1..])?; + if year == 0 || !(1..=12).contains(&month) || day == 0 || day > month_days(year, month) { + return Err(Rejected); + } + Ok(days_from_civil(year, month, day)) +} + +fn clock_micros(clock: &str) -> Result<(i128, i128), Rejected> { + let (clock, offset_seconds) = split_offset(clock)?; + let (hms, fraction) = clock + .split_once('.') + .map_or((clock, None), |(hms, fraction)| (hms, Some(fraction))); + let parts: Vec<&str> = hms.split(':').collect(); + if !CLOCK_PART_COUNTS.contains(&parts.len()) + || parts.iter().any(|part| part.len() != CLOCK_FIELD_LENGTH) + || (fraction.is_some() && parts.len() != CLOCK_PARTS_WITH_SECONDS) + { + return Err(Rejected); + } + let hour = number(parts[0])?; + let minute = number(parts[1])?; + let second = parts.get(2).map_or(Ok(0), |part| number(part))?; + if hour > MAX_HOUR || minute > MAX_MINUTE || second > MAX_MINUTE { + return Err(Rejected); + } + let fraction_micros = match fraction { + None => 0, + Some(digits) if FRACTION_DIGITS.contains(&digits.len()) => { + number(digits)? * 10_i128.pow(6 - u32::try_from(digits.len()).map_err(|_| Rejected)?) + } + Some(_) => return Err(Rejected), + }; + let seconds = hour * SECONDS_PER_HOUR + minute * SECONDS_PER_MINUTE + second; + Ok(( + seconds * MICROS_PER_SECOND + fraction_micros, + offset_seconds, + )) +} + +fn split_offset(clock: &str) -> Result<(&str, i128), Rejected> { + if let Some(clock) = clock.strip_suffix('Z') { + return Ok((clock, 0)); + } + let Some(sign_index) = clock.rfind(['+', '-']) else { + return Ok((clock, 0)); + }; + let offset = &clock[sign_index + 1..]; + let bytes = offset.as_bytes(); + if offset.len() != UTC_OFFSET_LENGTH || bytes[CLOCK_FIELD_LENGTH] != b':' { + return Err(Rejected); + } + let hours = number(&offset[..CLOCK_FIELD_LENGTH])?; + let minutes = number(&offset[CLOCK_FIELD_LENGTH + 1..])?; + if hours > MAX_HOUR || minutes > MAX_MINUTE { + return Err(Rejected); + } + let magnitude = hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE; + let sign = if clock.as_bytes()[sign_index] == b'-' { + -1 + } else { + 1 + }; + Ok((&clock[..sign_index], sign * magnitude)) +} + +fn number(digits: &str) -> Result { + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(Rejected); + } + digits.parse().map_err(|_| Rejected) +} + +fn month_days(year: i128, month: i128) -> i128 { + match month { + 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +fn days_from_civil(year: i128, month: i128, day: i128) -> i128 { + let shifted_year = if month <= LAST_SHIFTED_MONTH { + year - 1 + } else { + year + }; + let era = shifted_year.div_euclid(400); + let year_of_era = shifted_year - era * 400; + let shifted_month = (month + 9) % 12; + let day_of_year = (153 * shifted_month + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - CIVIL_EPOCH_OFFSET_DAYS +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs new file mode 100644 index 0000000000..46c7a50c5f --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/durations.rs @@ -0,0 +1,80 @@ +//! `Duration.parse` for ASCII text; other text is left to Python. + +use crate::model_validation::constants::DAY_UNIT_INDEX; +use crate::model_validation::models::Rejected; + +const MONTHS_PER_YEAR: u128 = 12; +const UNIT_SECONDS: [u128; 4] = [86_400, 3_600, 60, 1]; +const UNITS: [&str; 6] = ["y", "mo", "d", "h", "m", "s"]; + +/// A parsed non-zero duration. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Duration { + /// Years folded into months. + pub(crate) total_months: u128, + /// Days and smaller units as whole seconds. + pub(crate) fixed_seconds: u128, + /// Amounts by unit, in `UNITS` order. + amounts: [u128; 6], +} + +impl Duration { + /// Return whether only days are non-zero, like `units == {"d"}`. + pub(crate) fn is_whole_days(&self) -> bool { + self.amounts + .iter() + .enumerate() + .all(|(index, amount)| (index == DAY_UNIT_INDEX) == (*amount != 0)) + } + + /// Return the day amount. + pub(crate) fn days(&self) -> u128 { + self.amounts[DAY_UNIT_INDEX] + } +} + +/// Return `Duration.parse(text)`: a duration, `None` when it does not parse, or a rejection. +pub(crate) fn parse_duration(text: &str) -> Result, Rejected> { + if !text.is_ascii() || text.contains('\n') { + return Err(Rejected); + } + let mut amounts = [0_u128; 6]; + let mut next_unit = 0; + let mut rest = text; + while !rest.is_empty() { + let digits = rest.bytes().take_while(u8::is_ascii_digit).count(); + if digits == 0 { + return Ok(None); + } + let amount: u128 = rest[..digits].parse().map_err(|_| Rejected)?; + rest = &rest[digits..]; + let Some(unit) = (next_unit..UNITS.len()).find(|index| rest.starts_with(UNITS[*index])) + else { + return Ok(None); + }; + amounts[unit] = amount; + rest = &rest[UNITS[unit].len()..]; + next_unit = unit + 1; + } + let total_months = amounts[0] + .checked_mul(MONTHS_PER_YEAR) + .and_then(|months| months.checked_add(amounts[1])) + .ok_or(Rejected)?; + let fixed_seconds = amounts[DAY_UNIT_INDEX..] + .iter() + .zip(UNIT_SECONDS) + .try_fold(0_u128, |total, (amount, seconds)| { + amount + .checked_mul(seconds) + .and_then(|value| total.checked_add(value)) + }) + .ok_or(Rejected)?; + if total_months == 0 && fixed_seconds == 0 { + return Ok(None); + } + Ok(Some(Duration { + total_months, + fixed_seconds, + amounts, + })) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs new file mode 100644 index 0000000000..d1e8731d21 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs @@ -0,0 +1,335 @@ +//! `validate_incremental_config` for configs Python certainly accepts. + +use std::collections::HashSet; + +use crate::model_validation::_helpers::config::{ + ConfigView, has_config_value, one_of, optional_one_of, reject_case_insensitive_duplicates, + string_sequence, string_value, +}; +use crate::model_validation::_helpers::cursor_bounds::{cursor_bound_key, keys_ordered}; +use crate::model_validation::_helpers::durations::parse_duration; +use crate::model_validation::_helpers::microbatch::check_incremental_batching; +use crate::model_validation::_helpers::text::{python_lower, python_strip}; +use crate::model_validation::constants::{ + APPEND_STRATEGY, BOUNDED_REPLAY_PREFIX, CURSOR_GRAIN_BATCH_SIZES, CURSOR_INPUT_KEYS, + CURSOR_POLICY_DISABLED, CURSOR_TYPES, DELETE_INSERT_STRATEGY, EFFECTIVE_BATCH_SIZE, + ENFORCED_CONTRACT, FUTURE_CURSOR_ACTIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_STRATEGIES, + MERGE_STRATEGY, ON_SCHEMA_CHANGE_POLICIES, REPLAY_ON_CHANGE_ACTIONS, TABLE_MATERIALIZATION, + TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, ZERO_DAY_DURATION, +}; +use crate::model_validation::models::{ModelValidationFacts, Rejected}; +use crate::model_validation::types::Check; +use crate::types::{AuthoredNode, NodeKind}; + +/// The incremental config values `_resolve_incremental_config_values` reads. +pub(crate) struct IncrementalValues { + pub(crate) strategy: Option, + pub(crate) cursor: Option, + pub(crate) cursor_type: Option, + pub(crate) cursor_grain: Option, + pub(crate) lookback: Option, + pub(crate) incremental_mode: Option, + pub(crate) batch_size: Option, + pub(crate) microbatch_strategy: Option, + pub(crate) cursor_watermark_mode: Option, + pub(crate) cursor_start: Option, + pub(crate) cursor_end: Option, + pub(crate) unique_key: Option, + pub(crate) cursor_inputs: Option, + pub(crate) max_microbatches: Option, + pub(crate) microbatch_limit: Option, + pub(crate) merge_exclude_columns: Option, +} + +impl IncrementalValues { + fn read(config: &ConfigView) -> Result { + let node = |key: &str| config.get(key).cloned(); + Ok(Self { + strategy: config.string("incremental_strategy")?, + cursor: config.string("cursor")?, + cursor_type: config.string("cursor_type")?, + cursor_grain: config.string("cursor_grain")?, + lookback: config.string("lookback")?, + incremental_mode: config.string("incremental_mode")?, + batch_size: config.string("batch_size")?, + microbatch_strategy: config.string("microbatch_strategy")?, + cursor_watermark_mode: config.string("cursor_watermark_mode")?, + cursor_start: node("cursor_start"), + cursor_end: node("cursor_end"), + unique_key: node("unique_key"), + cursor_inputs: node("cursor_inputs"), + max_microbatches: node("max_microbatches"), + microbatch_limit: node("microbatch_limit"), + merge_exclude_columns: node("merge_exclude_columns"), + }) + } + + /// Return the `GRAIN_BATCH_SIZE` duration for an `effective` batch size, else the batch size. + pub(crate) fn effective_batch_size(&self) -> Option<&str> { + let batch_size = self.batch_size.as_deref()?; + if batch_size != EFFECTIVE_BATCH_SIZE { + return Some(batch_size); + } + let Some(grain) = self.cursor_grain.as_deref() else { + return Some(batch_size); + }; + CURSOR_GRAIN_BATCH_SIZES + .iter() + .find(|(name, _)| *name == grain) + .map_or(Some(batch_size), |(_, size)| Some(size)) + } +} + +/// Accept the incremental rules for one model's layered config. +pub(crate) fn check_incremental( + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Check { + let materialized = config.string("materialized")?; + let materialized = materialized.as_deref(); + if matches!( + materialized, + Some(TABLE_MATERIALIZATION | VIEW_MATERIALIZATION) + ) && config.any_present(&CURSOR_INPUT_KEYS) + { + return Err(Rejected); + } + if materialized != Some(INCREMENTAL_MATERIALIZATION) { + return Ok(()); + } + let values = IncrementalValues::read(config)?; + check_core(config, &values)?; + check_incremental_batching(&values, config, facts)?; + check_write_strategy(config, &values)?; + check_contract_columns(config, &values, facts)?; + if values.lookback.is_some() && values.cursor.is_none() { + return Err(Rejected); + } + Ok(()) +} + +fn check_core(config: &ConfigView, values: &IncrementalValues) -> Check { + if let Some(value) = config.get("on_schema_change") { + accept_text(value, |text| Ok(one_of(text, &ON_SCHEMA_CHANGE_POLICIES)))?; + } + if let Some(value) = config.get("replay_on_change") { + accept_text(value, valid_replay_on_change)?; + } + if !optional_one_of(values.strategy.as_deref(), &INCREMENTAL_STRATEGIES) { + return Err(Rejected); + } + check_cursor_rules(config, values)?; + check_cursor_safety(config, values) +} + +fn accept_text( + value: &N, + valid: impl FnOnce(&str) -> Result, +) -> Check { + if valid(&string_value(value)?)? { + Ok(()) + } else { + Err(Rejected) + } +} + +fn valid_replay_on_change(text: &str) -> Result { + if one_of(text, &REPLAY_ON_CHANGE_ACTIONS) { + return Ok(true); + } + match text.strip_prefix(BOUNDED_REPLAY_PREFIX) { + Some(duration) => Ok(parse_duration(python_strip(duration)?)?.is_some()), + None => Ok(false), + } +} + +fn check_cursor_rules( + config: &ConfigView, + values: &IncrementalValues, +) -> Check { + let cursor = values.cursor.as_deref(); + let cursor_type = values.cursor_type.as_deref(); + let grain = values.cursor_grain.as_deref(); + if cursor.is_some() && cursor_type.is_none() { + return Err(Rejected); + } + if cursor_type.is_some_and(|value| !one_of(value, &CURSOR_TYPES)) { + return Err(Rejected); + } + if grain.is_some() + && (cursor_type != Some(TIMESTAMP_CURSOR) + || !CURSOR_GRAIN_BATCH_SIZES + .iter() + .any(|(name, _)| Some(*name) == grain)) + { + return Err(Rejected); + } + if cursor.is_some() && cursor_type == Some(TIMESTAMP_CURSOR) && grain.is_none() { + return Err(Rejected); + } + let bounds = [values.cursor_start.as_ref(), values.cursor_end.as_ref()]; + if bounds.iter().any(Option::is_some) && (cursor.is_none() || cursor_type.is_none()) { + return Err(Rejected); + } + if let Some(cursor_type) = cursor_type { + let keys = bounds + .iter() + .flatten() + .map(|bound| cursor_bound_key(*bound, cursor_type)) + .collect::, _>>()?; + if let [start, end] = keys[..] + && !keys_ordered(start, end, cursor_type) + { + return Err(Rejected); + } + } + match config + .get("append_cursor_inclusive") + .map(AuthoredNode::kind) + { + None => Ok(()), + Some(NodeKind::Bool(_)) + if values.strategy.as_deref() == Some(APPEND_STRATEGY) && cursor.is_some() => + { + Ok(()) + } + Some(_) => Err(Rejected), + } +} + +fn check_cursor_safety( + config: &ConfigView, + values: &IncrementalValues, +) -> Check { + let durations = ["cursor_start_max_ahead", "cursor_future_max_distance"]; + let actions = ["cursor_start_max_action", "cursor_future_action"]; + if values.cursor.is_none() && (config.any_present(&durations) || config.any_present(&actions)) { + return Err(Rejected); + } + for key in durations { + if let Some(value) = config.get(key) { + accept_text(value, |text| { + Ok(text == CURSOR_POLICY_DISABLED + || text == ZERO_DAY_DURATION + || parse_duration(text)?.is_some()) + })?; + } + } + for key in actions { + if let Some(value) = config.get(key) { + accept_text(value, |text| Ok(one_of(text, &FUTURE_CURSOR_ACTIONS)))?; + } + } + Ok(()) +} + +fn check_write_strategy( + config: &ConfigView, + values: &IncrementalValues, +) -> Check { + let strategy = values.strategy.as_deref(); + let has_unique_key = has_config_value(values.unique_key.as_ref()); + if strategy == Some(DELETE_INSERT_STRATEGY) && values.cursor.is_none() && !has_unique_key { + return Err(Rejected); + } + if strategy == Some(MERGE_STRATEGY) && !has_unique_key { + return Err(Rejected); + } + if let Some(excluded) = &values.merge_exclude_columns { + let excluded = non_empty_strings(excluded)?; + if strategy != Some(MERGE_STRATEGY) { + return Err(Rejected); + } + reject_case_insensitive_duplicates(&excluded)?; + let unique_columns = string_sequence(values.unique_key.as_ref())? + .iter() + .map(|column| python_lower(column)) + .collect::, _>>()?; + for column in &excluded { + if unique_columns.contains(&python_lower(column)?) { + return Err(Rejected); + } + } + } + match config.get("full_refresh").map(AuthoredNode::kind) { + None | Some(NodeKind::Bool(_)) => Ok(()), + Some(_) => Err(Rejected), + } +} + +fn non_empty_strings(value: &N) -> Result, Rejected> { + if !matches!(value.kind(), NodeKind::List | NodeKind::Tuple) { + return Err(Rejected); + } + value + .items() + .iter() + .map(|item| string_value(item).and_then(non_empty)) + .collect() +} + +fn non_empty(text: String) -> Result { + if text.is_empty() { + Err(Rejected) + } else { + Ok(text) + } +} + +fn check_contract_columns( + config: &ConfigView, + values: &IncrementalValues, + facts: &ModelValidationFacts<'_>, +) -> Check { + let Some(declared) = contract_declared_columns(config, facts) else { + return Ok(()); + }; + let named = values + .cursor + .iter() + .cloned() + .chain(string_sequence(values.unique_key.as_ref())?) + .chain(string_sequence(values.merge_exclude_columns.as_ref())?); + require_declared(named, &declared) +} + +/// Return `_contract_declared_column_names` when the contract is enforced. +pub(crate) fn contract_declared_columns( + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Option> { + if !config.equals("contract", ENFORCED_CONTRACT) { + return None; + } + let mut names: HashSet = facts + .declared_columns + .unwrap_or_default() + .iter() + .cloned() + .collect(); + if let Some(columns) = config + .get("columns") + .filter(|value| value.kind() == NodeKind::Map) + { + names.extend( + columns + .entries() + .iter() + .filter(|(key, _)| key.kind() == NodeKind::Str) + .filter_map(|(key, _)| key.text()), + ); + } + Some(names) +} + +/// Reject any column name the enforced contract does not declare. +pub(crate) fn require_declared( + mut names: impl Iterator, + declared: &HashSet, +) -> Check { + if names.all(|name| declared.contains(&name)) { + Ok(()) + } else { + Err(Rejected) + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs new file mode 100644 index 0000000000..1af4ad3848 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -0,0 +1,187 @@ +//! Contract, materialization, storage, migration and placeholder rules Python certainly accepts. + +use std::collections::BTreeSet; + +use crate::model_validation::_helpers::config::{ConfigView, one_of, string_value}; +use crate::model_validation::_helpers::durations::parse_duration; +use crate::model_validation::_helpers::text::{python_lower, python_strip}; +use crate::model_validation::constants::{ + BUILTIN_MATERIALIZATIONS, CONTRACT_POLICIES, CUSTOM_MATERIALIZATION_DISALLOWED_KEYS, + HISTORY_MATERIALIZATIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_ONLY_KEYS, + OLD_NAME_VIEW_KEY, PLACEHOLDER_SIGIL, TABLE_BACKED_MATERIALIZATIONS, VIEW_MATERIALIZATION, +}; +use crate::model_validation::models::{ModelValidationFacts, ProjectValidationFacts, Rejected}; +use crate::model_validation::types::Check; +use crate::types::{AuthoredNode, NodeKind}; + +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const MIGRATE_FORCE_KEY: &str = "migrate_force"; + +/// Accept `validate_microbatch_project_capability`. +pub(crate) fn check_project_capability( + config: &ConfigView, + project: &ProjectValidationFacts, +) -> Check { + let Some(concurrency) = config.get("batch_concurrency") else { + return Ok(()); + }; + match (concurrency.kind(), concurrency.integer()) { + (NodeKind::Int { .. }, Some(value)) if value > 1 && !project.microbatch_concurrency => { + Err(Rejected) + } + (NodeKind::Int { .. }, None) => Err(Rejected), + _ => Ok(()), + } +} + +/// Accept `validate_contract_config` and `validate_non_incremental_config`. +pub(crate) fn check_contract_and_incremental_keys( + config: &ConfigView, +) -> Check { + if let Some(contract) = config.get("contract") + && !one_of(&string_value(contract)?, &CONTRACT_POLICIES) + { + return Err(Rejected); + } + if config.string("materialized")?.as_deref() != Some(INCREMENTAL_MATERIALIZATION) + && config.any_present(&INCREMENTAL_ONLY_KEYS) + { + return Err(Rejected); + } + Ok(()) +} + +/// Accept `validate_custom_materialization_config`. +pub(crate) fn check_custom_materialization( + config: &ConfigView, + project: &ProjectValidationFacts, +) -> Check { + let Some(materialized) = config.string("materialized")? else { + return Ok(()); + }; + if one_of(&materialized, &BUILTIN_MATERIALIZATIONS) { + return Ok(()); + } + if !project.custom_materializations.contains(&materialized) + || config.any_present(&CUSTOM_MATERIALIZATION_DISALLOWED_KEYS) + { + return Err(Rejected); + } + Ok(()) +} + +/// Accept `validate_storage_policies`. +pub(crate) fn check_storage_policies( + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Check { + if facts.retention_unmanaged && !facts.table_type_declared { + return Ok(()); + } + let materialized = config.string("materialized")?; + match materialized.as_deref() { + Some(VIEW_MATERIALIZATION) | None => Err(Rejected), + Some(value) if one_of(value, &TABLE_BACKED_MATERIALIZATIONS) => Ok(()), + Some(_) => Err(Rejected), + } +} + +/// Accept `validate_model_migration_config`. +pub(crate) fn check_migration( + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Check { + if config.contains_key(OLD_NAME_VIEW_KEY) { + let value = config.get(OLD_NAME_VIEW_KEY).ok_or(Rejected)?; + if value.kind() != NodeKind::Bool(false) { + let text = string_value(value)?; + if parse_duration(python_strip(&text)?)?.is_none() { + return Err(Rejected); + } + } + } + let migrate_from = config.get(MIGRATE_FROM_KEY); + let migrate_force = config.get(MIGRATE_FORCE_KEY); + if migrate_from.is_none() && migrate_force.is_none() { + return Ok(()); + } + let materialized = config.string("materialized")?.ok_or(Rejected)?; + if !one_of(&materialized, &TABLE_BACKED_MATERIALIZATIONS) + && materialized != VIEW_MATERIALIZATION + { + return Err(Rejected); + } + if migrate_force.is_some() && !one_of(&materialized, &HISTORY_MATERIALIZATIONS) { + return Err(Rejected); + } + let origin = string_value(migrate_from.ok_or(Rejected)?)?; + let origin = python_strip(&origin)?; + if origin.is_empty() || python_lower(origin)? == python_lower(facts.model_name)? { + return Err(Rejected); + } + match migrate_force.map(AuthoredNode::kind) { + None | Some(NodeKind::Bool(_)) => Ok(()), + Some(_) => Err(Rejected), + } +} + +/// Accept `validate_placeholder_config`. +pub(crate) fn check_placeholders( + config: &ConfigView, + project: &ProjectValidationFacts, + facts: &ModelValidationFacts<'_>, +) -> Check { + let materialized = config.string("materialized")?; + let custom = materialized + .as_ref() + .is_some_and(|value| project.custom_materializations.contains(value)); + let used = placeholder_names(facts.query_sql)?; + let declared = config.get("placeholders"); + if !custom { + return if used.is_empty() && declared.is_none() { + Ok(()) + } else { + Err(Rejected) + }; + } + let declared_names = match declared { + Some(value) if value.kind() == NodeKind::Map => value + .entries() + .iter() + .map(|(key, _)| string_value(key)) + .collect::, _>>()?, + _ => BTreeSet::new(), + }; + if declared_names == used { + Ok(()) + } else { + Err(Rejected) + } +} + +/// Return the names `re.findall(r"@@@(\w+)", sql)` finds, rejecting non-ASCII word candidates. +fn placeholder_names(sql: &str) -> Result, Rejected> { + let mut names: BTreeSet = BTreeSet::new(); + let mut rest = sql; + while let Some(start) = rest.find(PLACEHOLDER_SIGIL) { + let after = &rest[start + PLACEHOLDER_SIGIL.len()..]; + let length = after + .bytes() + .take_while(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + .count(); + if after[length..] + .chars() + .next() + .is_some_and(|next| !next.is_ascii()) + { + return Err(Rejected); + } + if length == 0 { + rest = &rest[start + 1..]; + continue; + } + names.insert(after[..length].to_owned()); + rest = &after[length..]; + } + Ok(names) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs new file mode 100644 index 0000000000..5606179318 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/microbatch.rs @@ -0,0 +1,291 @@ +//! `_validate_incremental_batching` for configs Python certainly accepts. + +use std::collections::HashSet; + +use crate::model_validation::_helpers::config::{ + ConfigView, non_blank_string, one_of, optional_one_of, string_value, +}; +use crate::model_validation::_helpers::durations::parse_duration; +use crate::model_validation::_helpers::incremental::IncrementalValues; +use crate::model_validation::constants::{ + CAP_FROM_START_ACTION, CURSOR_INPUT_ROLES, DELETE_INSERT_STRATEGY, EFFECTIVE_BATCH_SIZE, + FILTER_ROLE, INCREMENTAL_MODES, MERGE_STRATEGY, MICROBATCH_LIMIT_ACTIONS, + MICROBATCH_LIMIT_KEYS, MICROBATCH_MODE, MICROBATCH_STRATEGIES, REMOVED_CURSOR_INPUT_KEYS, + ROLLING_WINDOW_STRATEGY, TIMESTAMP_CURSOR, UNACCOUNTED_PARTITION_POLICIES, + WATERMARK_BLOCK_KEYS, WATERMARK_MODES, WATERMARK_ROLE, WATERMARK_STRATEGY, +}; +use crate::model_validation::models::{ModelValidationFacts, Rejected}; +use crate::model_validation::types::Check; +use crate::types::{AuthoredNode, NodeKind}; + +/// Accept the batching, microbatch and cursor input rules. +pub(crate) fn check_incremental_batching( + values: &IncrementalValues, + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Check { + let mode = values.incremental_mode.as_deref(); + if mode.is_some_and(|value| !one_of(value, &INCREMENTAL_MODES)) { + return Err(Rejected); + } + let microbatch = mode == Some(MICROBATCH_MODE); + if let Some(batch_size) = values.batch_size.as_deref() { + if !microbatch { + return Err(Rejected); + } + if batch_size == EFFECTIVE_BATCH_SIZE + && (values.cursor.is_none() + || values.cursor_type.as_deref() != Some(TIMESTAMP_CURSOR) + || values.cursor_grain.is_none()) + { + return Err(Rejected); + } + } + check_state_config(values, config, microbatch)?; + if config.any_present(&REMOVED_CURSOR_INPUT_KEYS) { + return Err(Rejected); + } + check_cursor_inputs(values, facts)?; + if let Some(limit) = microbatch_limit(values)? { + check_static_watermark_limit(values, limit)?; + } + Ok(()) +} + +fn check_state_config( + values: &IncrementalValues, + config: &ConfigView, + microbatch: bool, +) -> Check { + if let Some(concurrency) = config.get("batch_concurrency") { + let concurrency = concurrency.integer().ok_or(Rejected)?; + if concurrency <= 0 + || (concurrency > 1 + && (!microbatch || values.strategy.as_deref() != Some(DELETE_INSERT_STRATEGY))) + { + return Err(Rejected); + } + } + if let Some(policy) = config.get("unaccounted_partition_policy") + && (!one_of(&string_value(policy)?, &UNACCOUNTED_PARTITION_POLICIES) || !microbatch) + { + return Err(Rejected); + } + Ok(()) +} + +fn check_cursor_inputs( + values: &IncrementalValues, + facts: &ModelValidationFacts<'_>, +) -> Check { + let microbatch = values.incremental_mode.as_deref() == Some(MICROBATCH_MODE); + let strategy = values.microbatch_strategy.as_deref(); + let reference_count = facts.references.len(); + if strategy.is_some() != microbatch + || strategy.is_some_and(|value| !one_of(value, &MICROBATCH_STRATEGIES)) + { + return Err(Rejected); + } + if strategy == Some(ROLLING_WINDOW_STRATEGY) + && values.cursor_type.as_deref() != Some(TIMESTAMP_CURSOR) + { + return Err(Rejected); + } + let inputs = values.cursor_inputs.as_ref(); + if inputs.is_some() && values.cursor.is_none() { + return Err(Rejected); + } + if inputs.is_none() + && (strategy == Some(WATERMARK_STRATEGY) + || (strategy == Some(ROLLING_WINDOW_STRATEGY) && reference_count > 0)) + { + return Err(Rejected); + } + let input_names: HashSet<&str> = facts + .references + .iter() + .map(|reference| reference.name.as_str()) + .collect(); + if let Some(inputs) = inputs { + if strategy == Some(WATERMARK_STRATEGY) { + check_watermark_inputs(inputs, &input_names)?; + } else { + check_input_map(inputs, &input_names)?; + } + } + let watermark_mode = values.cursor_watermark_mode.as_deref(); + if (strategy == Some(WATERMARK_STRATEGY) && !optional_one_of(watermark_mode, &WATERMARK_MODES)) + || (strategy != Some(WATERMARK_STRATEGY) && watermark_mode.is_some()) + { + return Err(Rejected); + } + if let Some(limit) = &values.max_microbatches + && (limit.integer().is_none_or(|value| value < 1) || strategy != Some(WATERMARK_STRATEGY)) + { + return Err(Rejected); + } + if values.cursor.is_some() && reference_count > 1 && inputs.is_none() { + return Err(Rejected); + } + Ok(()) +} + +fn check_input_map(inputs: &N, input_names: &HashSet<&str>) -> Check { + let entries = map_entries(inputs)?; + if entries.is_empty() { + return Err(Rejected); + } + for (relation, column) in &entries { + let relation = non_blank_string(relation)?; + non_blank_string(column)?; + if !input_names.contains(relation.as_str()) { + return Err(Rejected); + } + } + Ok(()) +} + +fn check_watermark_inputs(inputs: &N, input_names: &HashSet<&str>) -> Check { + let entries = map_entries(inputs)?; + if entries.is_empty() { + return Err(Rejected); + } + let mut has_watermark = false; + for (relation, block) in &entries { + let relation = non_blank_string(relation)?; + let block = keyed_entries(block, &WATERMARK_BLOCK_KEYS)?; + non_blank_string(&block[0])?; + let roles = &block[1]; + if roles.kind() != NodeKind::List { + return Err(Rejected); + } + let roles = roles + .items() + .iter() + .map(string_value) + .collect::, _>>()?; + let distinct: HashSet<&String> = roles.iter().collect(); + if roles.is_empty() + || roles.iter().any(|role| !one_of(role, &CURSOR_INPUT_ROLES)) + || distinct.len() != roles.len() + { + return Err(Rejected); + } + has_watermark |= roles.iter().any(|role| role == WATERMARK_ROLE); + if roles.iter().any(|role| role == FILTER_ROLE) && !input_names.contains(relation.as_str()) + { + return Err(Rejected); + } + } + if has_watermark { Ok(()) } else { Err(Rejected) } +} + +fn map_entries(value: &N) -> Result, Rejected> { + if value.kind() == NodeKind::Map { + Ok(value.entries()) + } else { + Err(Rejected) + } +} + +/// Return a mapping's values in `keys` order when its keys are exactly `keys`. +fn keyed_entries(value: &N, keys: &[&str]) -> Result, Rejected> { + let entries = map_entries(value)?; + if entries.len() != keys.len() { + return Err(Rejected); + } + keys.iter().map(|key| entry_value(&entries, key)).collect() +} + +fn entry_value(entries: &[(N, N)], key: &str) -> Result { + for (name, item) in entries { + if name.kind() == NodeKind::Str && name.is_text(key) { + return Ok(item.clone()); + } + } + Err(Rejected) +} + +/// The resolved batch limit and its action, as `_validate_model_microbatch_limit` returns them. +struct MicrobatchLimit { + max_batches: i64, + cap_from_start: bool, +} + +fn microbatch_limit( + values: &IncrementalValues, +) -> Result, Rejected> { + let Some(limit) = &values.microbatch_limit else { + return Ok(values + .max_microbatches + .as_ref() + .and_then(AuthoredNode::integer) + .map(|max_batches| MicrobatchLimit { + max_batches, + cap_from_start: false, + })); + }; + if values.max_microbatches.is_some() { + return Err(Rejected); + } + let block = keyed_entries(limit, &MICROBATCH_LIMIT_KEYS)?; + let max_batches = block[0] + .integer() + .filter(|value| *value >= 1) + .ok_or(Rejected)?; + let action = string_value(&block[1])?; + if !one_of(&action, &MICROBATCH_LIMIT_ACTIONS) + || values.microbatch_strategy.as_deref() != Some(WATERMARK_STRATEGY) + { + return Err(Rejected); + } + Ok(Some(MicrobatchLimit { + max_batches, + cap_from_start: action == CAP_FROM_START_ACTION, + })) +} + +fn check_static_watermark_limit( + values: &IncrementalValues, + limit: MicrobatchLimit, +) -> Check { + let batch_size = values.effective_batch_size(); + let rewrites_lookback = limit.cap_from_start + && matches!( + values.strategy.as_deref(), + Some(DELETE_INSERT_STRATEGY | MERGE_STRATEGY) + ); + let lookback = values + .lookback + .as_deref() + .or(batch_size.filter(|_| rewrites_lookback)); + let (Some(lookback), Some(batch_size)) = (lookback, batch_size) else { + return Ok(()); + }; + let (Some(lookback), Some(batch)) = (parse_duration(lookback)?, parse_duration(batch_size)?) + else { + return Ok(()); + }; + let lookback_batches = if lookback.total_months == 0 && batch.total_months == 0 { + if batch.fixed_seconds == 0 { + return Ok(()); + } + lookback.fixed_seconds.div_ceil(batch.fixed_seconds) + } else if lookback.fixed_seconds == 0 && batch.fixed_seconds == 0 { + if batch.total_months == 0 { + return Ok(()); + } + lookback.total_months.div_ceil(batch.total_months) + } else if limit.cap_from_start { + return Err(Rejected); + } else { + return Ok(()); + }; + let required = lookback_batches + 1 + u128::from(limit.cap_from_start); + let max_batches = u128::try_from(limit.max_batches).map_err(|_| Rejected)?; + if max_batches < required { + Err(Rejected) + } else { + Ok(()) + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs new file mode 100644 index 0000000000..bb9420ceec --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod config; +pub(crate) mod cursor_bounds; +pub(crate) mod durations; +pub(crate) mod incremental; +pub(crate) mod materialization; +pub(crate) mod microbatch; +pub(crate) mod references; +pub(crate) mod snapshot; +pub(crate) mod text; diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs new file mode 100644 index 0000000000..5bc4380f87 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/references.rs @@ -0,0 +1,32 @@ +//! `validate_model_references` for references every discovered name resolves. + +use crate::model_validation::constants::{ + REF_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, UDF_KIND, +}; +use crate::model_validation::models::{ModelReference, ProjectValidationFacts, Rejected}; +use crate::model_validation::types::Check; + +/// Accept references that name discovered resources of their own kind. +pub(crate) fn check_references( + references: &[ModelReference], + project: &ProjectValidationFacts, +) -> Check { + let resolves = |reference: &ModelReference| { + let name = &reference.name; + match reference.kind.as_str() { + REF_KIND => project.models.contains(name), + SEED_KIND => project.seeds.contains(name), + SOURCE_KIND => project.sources.contains(name), + UDF_KIND => project.functions.contains(name) && !project.table_functions.contains(name), + TABLE_FUNCTION_KIND => { + project.functions.contains(name) && project.table_functions.contains(name) + } + _ => false, + } + }; + if references.iter().all(resolves) { + Ok(()) + } else { + Err(Rejected) + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs new file mode 100644 index 0000000000..555298802b --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs @@ -0,0 +1,115 @@ +//! `validate_snapshot_config` for configs Python certainly accepts. + +use crate::model_validation::_helpers::config::{ + ConfigView, has_config_value, one_of, string_sequence, +}; +use crate::model_validation::_helpers::incremental::{contract_declared_columns, require_declared}; +use crate::model_validation::_helpers::text::python_lower; +use crate::model_validation::constants::{ + APPEND_NEW_COLUMNS_POLICY, CHANGES_HISTORICAL_INPUT, CHECK_SNAPSHOT_STRATEGY, + ENFORCED_CONTRACT, HISTORICAL_INPUTS, INITIAL_VALID_FROM_VALUES, SNAPSHOT_DISALLOWED_KEYS, + SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_MATERIALIZATION, SNAPSHOT_SCHEMA_CHANGE_POLICIES, + SNAPSHOT_STRATEGIES, SQL_WILDCARD, TIMESTAMP_SNAPSHOT_STRATEGY, +}; +use crate::model_validation::models::{ModelValidationFacts, Rejected}; +use crate::model_validation::types::Check; +use crate::types::{AuthoredNode, NodeKind}; + +const UPDATED_AT: &str = "updated_at"; +const OBSERVED_AT: &str = "observed_at"; + +/// Accept the snapshot rules for one model's layered config. +pub(crate) fn check_snapshot( + config: &ConfigView, + facts: &ModelValidationFacts<'_>, +) -> Check { + if config.string("materialized")?.as_deref() != Some(SNAPSHOT_MATERIALIZATION) { + return Ok(()); + } + let strategy = config.string("snapshot_strategy")?; + let updated_at = config.string(UPDATED_AT)?; + let observed_at = config.string(OBSERVED_AT)?; + let historical_input = config.string("historical_input")?; + let initial_valid_from = config.string("initial_valid_from")?; + let full_refresh = config.string("snapshot_full_refresh")?; + let schema_change = config.string("snapshot_schema_change")?; + let valid_from_column = config.string("valid_from_column")?; + let valid_to_column = config.string("valid_to_column")?; + let unique_key = config.get("unique_key"); + let check_columns = config.get("check_columns"); + let strategy = strategy.as_deref(); + let historical_input = historical_input.as_deref(); + let changes = historical_input == Some(CHANGES_HISTORICAL_INPUT); + let check = strategy == Some(CHECK_SNAPSHOT_STRATEGY); + let timestamp = strategy == Some(TIMESTAMP_SNAPSHOT_STRATEGY); + if config.any_present(&SNAPSHOT_DISALLOWED_KEYS) + || !has_config_value(unique_key) + || !strategy.is_some_and(|value| one_of(value, &SNAPSHOT_STRATEGIES)) + || (timestamp && updated_at.is_none()) + || (check && !has_config_value(check_columns)) + || (check && check_columns.is_some_and(mixes_wildcard)) + || (historical_input.is_some() && observed_at.is_none()) + || historical_input.is_some_and(|value| !one_of(value, &HISTORICAL_INPUTS)) + || (observed_at.is_some() && timestamp && historical_input.is_none()) + || (check && changes) + { + return Err(Rejected); + } + match config + .get("invalidate_hard_deletes") + .map(AuthoredNode::kind) + { + None | Some(NodeKind::Bool(false)) => {} + Some(NodeKind::Bool(true)) if !changes => {} + Some(_) => return Err(Rejected), + } + let initial_valid_from = initial_valid_from.as_deref(); + if initial_valid_from.is_some_and(|value| !one_of(value, &INITIAL_VALID_FROM_VALUES)) + || (initial_valid_from == Some(UPDATED_AT) && updated_at.is_none()) + || (initial_valid_from == Some(OBSERVED_AT) && observed_at.is_none()) + || full_refresh + .as_deref() + .is_some_and(|value| !one_of(value, &SNAPSHOT_FULL_REFRESH_POLICIES)) + { + return Err(Rejected); + } + if let Some(schema_change) = schema_change.as_deref() + && (!one_of(schema_change, &SNAPSHOT_SCHEMA_CHANGE_POLICIES) + || (config.equals("contract", ENFORCED_CONTRACT) + && schema_change == APPEND_NEW_COLUMNS_POLICY)) + { + return Err(Rejected); + } + if let (Some(valid_from), Some(valid_to)) = (&valid_from_column, &valid_to_column) + && python_lower(valid_from)? == python_lower(valid_to)? + { + return Err(Rejected); + } + let Some(declared) = contract_declared_columns(config, facts) else { + return Ok(()); + }; + let checked_columns = if check_columns.is_some_and(is_wildcard_only) { + Vec::new() + } else { + string_sequence(check_columns)? + }; + let named = string_sequence(unique_key)? + .into_iter() + .chain(updated_at) + .chain(observed_at) + .chain(checked_columns); + require_declared(named, &declared) +} + +fn mixes_wildcard(columns: &N) -> bool { + if columns.kind() != NodeKind::List { + return false; + } + let items = columns.items(); + items.len() != 1 && items.iter().any(|item| item.is_text(SQL_WILDCARD)) +} + +fn is_wildcard_only(columns: &N) -> bool { + columns.kind() == NodeKind::List + && matches!(&columns.items()[..], [item] if item.is_text(SQL_WILDCARD)) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs new file mode 100644 index 0000000000..378f679bb2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/text.rs @@ -0,0 +1,25 @@ +//! ASCII text operations that match Python's `str` methods; other text is left to Python. + +use crate::model_validation::models::Rejected; + +/// Return Python's `text.strip()` for ASCII text, or reject other text. +pub(crate) fn python_strip(text: &str) -> Result<&str, Rejected> { + if text.is_ascii() { + Ok(text.trim_matches(|character: char| is_python_space(character as u8))) + } else { + Err(Rejected) + } +} + +/// Return Python's `text.lower()` for ASCII text, or reject other text. +pub(crate) fn python_lower(text: &str) -> Result { + if text.is_ascii() { + Ok(text.to_ascii_lowercase()) + } else { + Err(Rejected) + } +} + +fn is_python_space(byte: u8) -> bool { + matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') +} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs new file mode 100644 index 0000000000..a92e79efd2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -0,0 +1,215 @@ +//! Config keys and vocabularies the Python model validators accept. + +/// Built-in materializations that never name a custom materialization. +pub const BUILTIN_MATERIALIZATIONS: [&str; 4] = ["view", "table", "incremental", "snapshot"]; +/// Materializations that own a physical managed table. +pub const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; +/// Materializations that keep history across runs. +pub const HISTORY_MATERIALIZATIONS: [&str; 2] = ["incremental", "snapshot"]; +/// The view materialization. +pub const VIEW_MATERIALIZATION: &str = "view"; +/// The table materialization. +pub const TABLE_MATERIALIZATION: &str = "table"; +/// The incremental materialization. +pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +/// The snapshot materialization. +pub const SNAPSHOT_MATERIALIZATION: &str = "snapshot"; + +/// `IncrementalStrategy` values. +pub const INCREMENTAL_STRATEGIES: [&str; 3] = ["append", "delete_insert", "merge"]; +/// The append incremental strategy. +pub const APPEND_STRATEGY: &str = "append"; +/// The delete-insert incremental strategy. +pub const DELETE_INSERT_STRATEGY: &str = "delete_insert"; +/// The merge incremental strategy. +pub const MERGE_STRATEGY: &str = "merge"; +/// `IncrementalMode` values. +pub const INCREMENTAL_MODES: [&str; 2] = ["full", "microbatch"]; +/// The microbatch incremental mode. +pub const MICROBATCH_MODE: &str = "microbatch"; +/// `MicrobatchStrategy` values. +pub const MICROBATCH_STRATEGIES: [&str; 2] = ["rolling_window", "watermark"]; +/// The rolling-window microbatch strategy. +pub const ROLLING_WINDOW_STRATEGY: &str = "rolling_window"; +/// The watermark microbatch strategy. +pub const WATERMARK_STRATEGY: &str = "watermark"; +/// `CursorWatermarkMode` values. +pub const WATERMARK_MODES: [&str; 2] = ["all", "any"]; +/// `CursorInputRole` values. +pub const CURSOR_INPUT_ROLES: [&str; 2] = ["filter", "watermark"]; +/// The filter cursor input role. +pub const FILTER_ROLE: &str = "filter"; +/// The watermark cursor input role. +pub const WATERMARK_ROLE: &str = "watermark"; +/// `CursorType` values. +pub const CURSOR_TYPES: [&str; 2] = ["timestamp", "integer"]; +/// The timestamp cursor type. +pub const TIMESTAMP_CURSOR: &str = "timestamp"; +/// The integer cursor type. +pub const INTEGER_CURSOR: &str = "integer"; +/// `CursorGrain` values with their `GRAIN_BATCH_SIZE` durations. +pub const CURSOR_GRAIN_BATCH_SIZES: [(&str, &str); 6] = [ + ("second", "1s"), + ("minute", "1m"), + ("hour", "1h"), + ("day", "1d"), + ("month", "1mo"), + ("year", "1y"), +]; +/// `FutureCursorAction` values. +pub const FUTURE_CURSOR_ACTIONS: [&str; 2] = ["cap", "error"]; +/// `MicrobatchLimitAction` values. +pub const MICROBATCH_LIMIT_ACTIONS: [&str; 4] = ["cap_from_end", "cap_from_start", "error", "warn"]; +/// The cap-from-start microbatch limit action. +pub const CAP_FROM_START_ACTION: &str = "cap_from_start"; +/// Accepted `unaccounted_partition_policy` values. +pub const UNACCOUNTED_PARTITION_POLICIES: [&str; 3] = + ["synthesize", "recover_empty", "recover_all"]; +/// `ContractPolicy` values. +pub const CONTRACT_POLICIES: [&str; 2] = ["none", "enforced"]; +/// The enforced contract policy. +pub const ENFORCED_CONTRACT: &str = "enforced"; +/// `OnSchemaChange` values. +pub const ON_SCHEMA_CHANGE_POLICIES: [&str; 4] = + ["ignore", "fail", "append_new_columns", "sync_all_columns"]; +/// Unbounded `replay_on_change` values. +pub const REPLAY_ON_CHANGE_ACTIONS: [&str; 2] = ["forward", "full"]; +/// The prefix of a bounded `replay_on_change` value. +pub const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; +/// `SnapshotStrategy` values. +pub const SNAPSHOT_STRATEGIES: [&str; 2] = ["timestamp", "check"]; +/// The timestamp snapshot strategy. +pub const TIMESTAMP_SNAPSHOT_STRATEGY: &str = "timestamp"; +/// The check snapshot strategy. +pub const CHECK_SNAPSHOT_STRATEGY: &str = "check"; +/// `HistoricalInput` values. +pub const HISTORICAL_INPUTS: [&str; 2] = ["snapshot", "changes"]; +/// The changes historical input. +pub const CHANGES_HISTORICAL_INPUT: &str = "changes"; +/// `InitialValidFrom` values. +pub const INITIAL_VALID_FROM_VALUES: [&str; 3] = ["updated_at", "observed_at", "execution_time"]; +/// `SnapshotFullRefreshPolicy` values. +pub const SNAPSHOT_FULL_REFRESH_POLICIES: [&str; 3] = ["deny", "require_confirmation", "allow"]; +/// `SnapshotSchemaChangePolicy` values. +pub const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] = + ["deny", "require_confirmation", "append_new_columns"]; +/// The snapshot schema-change policy that conflicts with an enforced contract. +pub const APPEND_NEW_COLUMNS_POLICY: &str = "append_new_columns"; +/// The duration-disabled cursor policy value. +pub const CURSOR_POLICY_DISABLED: &str = "disabled"; +/// The zero-day cursor duration. +pub const ZERO_DAY_DURATION: &str = "0d"; +/// The `batch_size` token that follows the cursor grain. +pub const EFFECTIVE_BATCH_SIZE: &str = "effective"; +/// The SQL wildcard column. +pub const SQL_WILDCARD: &str = "*"; +/// The `@@@name` placeholder sigil. +pub const PLACEHOLDER_SIGIL: &str = "@@@"; + +/// Keys that only incremental models may set. +pub const INCREMENTAL_ONLY_KEYS: [&str; 5] = [ + "on_schema_change", + "replay_on_change", + "append_cursor_inclusive", + "merge_exclude_columns", + "full_refresh", +]; +/// Cursor input keys that require a cursor-based incremental model. +pub const CURSOR_INPUT_KEYS: [&str; 3] = [ + "cursor_inputs", + "cursor_filter_inputs", + "cursor_watermark_inputs", +]; +/// Cursor input keys that have been removed in favour of `cursor_inputs`. +pub const REMOVED_CURSOR_INPUT_KEYS: [&str; 2] = + ["cursor_filter_inputs", "cursor_watermark_inputs"]; +/// Keys a snapshot model may not set. +pub const SNAPSHOT_DISALLOWED_KEYS: [&str; 16] = [ + "incremental_strategy", + "incremental_mode", + "append_cursor_inclusive", + "batch_size", + "cursor", + "cursor_type", + "cursor_grain", + "cursor_inputs", + "cursor_filter_inputs", + "cursor_watermark_inputs", + "cursor_end", + "microbatch_strategy", + "cursor_watermark_mode", + "max_microbatches", + "microbatch_limit", + "lookback", +]; +/// Keys a custom materialization may not set. +pub const CUSTOM_MATERIALIZATION_DISALLOWED_KEYS: [&str; 17] = [ + "on_schema_change", + "incremental_strategy", + "incremental_mode", + "append_cursor_inclusive", + "batch_size", + "cursor", + "cursor_type", + "cursor_grain", + "cursor_inputs", + "cursor_filter_inputs", + "cursor_watermark_inputs", + "cursor_end", + "microbatch_strategy", + "cursor_watermark_mode", + "max_microbatches", + "microbatch_limit", + "lookback", +]; +/// The `microbatch_limit` keys, in no particular order. +pub const MICROBATCH_LIMIT_KEYS: [&str; 2] = ["max_batches", "action"]; +/// The watermark `cursor_inputs` block keys, in no particular order. +pub const WATERMARK_BLOCK_KEYS: [&str; 2] = ["column", "roles"]; +/// The config key naming the old-name compatibility view. +pub const OLD_NAME_VIEW_KEY: &str = "old_name_view"; + +/// `SqlReferenceKind` values a model may reference. +pub const REF_KIND: &str = "ref"; +/// A seed reference. +pub const SEED_KIND: &str = "seed"; +/// A source reference. +pub const SOURCE_KIND: &str = "source"; +/// A scalar SQL function reference. +pub const UDF_KIND: &str = "udf"; +/// A table function reference. +pub const TABLE_FUNCTION_KIND: &str = "table_fn"; + +/// The `inherit` storage policy value. +pub const INHERIT_POLICY: &str = "inherit"; +/// The `disabled` retention value. +pub const DISABLED_RETENTION: &str = "disabled"; +/// The permanent table type. +pub const PERMANENT_TABLE_TYPE: &str = "permanent"; +/// The transient table type. +pub const TRANSIENT_TABLE_TYPE: &str = "transient"; + +/// The length of an ISO `YYYY-MM-DD` date. +pub const ISO_DATE_LENGTH: usize = 10; +/// Byte offsets of the separators in an ISO date. +pub const ISO_DATE_SEPARATOR_OFFSETS: [usize; 2] = [4, 7]; +/// The length of a two-digit clock field. +pub const CLOCK_FIELD_LENGTH: usize = 2; +/// Clock part counts `HH:MM` and `HH:MM:SS`. +pub const CLOCK_PART_COUNTS: [usize; 2] = [2, 3]; +/// The clock part count that may carry fractional seconds. +pub const CLOCK_PARTS_WITH_SECONDS: usize = 3; +/// The latest hour of a day. +pub const MAX_HOUR: i128 = 23; +/// The latest minute of an hour, and second of a minute. +pub const MAX_MINUTE: i128 = 59; +/// The length of a `HH:MM` UTC offset. +pub const UTC_OFFSET_LENGTH: usize = 5; +/// The first year Python's `datetime` represents. +pub const MIN_YEAR: i128 = 1; +/// The last year Python's `datetime` represents. +pub const MAX_YEAR: i128 = 9999; +/// The last month whose civil-day computation shifts the year back. +pub const LAST_SHIFTED_MONTH: i128 = 2; +/// The position of days among duration units. +pub const DAY_UNIT_INDEX: usize = 2; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/accept_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/accept_model_config.rs new file mode 100644 index 0000000000..c166ecd066 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/accept_model_config.rs @@ -0,0 +1,31 @@ +//! Accept a model's effective config when every Python model validator would. + +use crate::model_validation::_helpers::config::ConfigView; +use crate::model_validation::_helpers::incremental::check_incremental; +use crate::model_validation::_helpers::materialization::{ + check_contract_and_incremental_keys, check_custom_materialization, check_migration, + check_placeholders, check_project_capability, check_storage_policies, +}; +use crate::model_validation::_helpers::references::check_references; +use crate::model_validation::_helpers::snapshot::check_snapshot; +use crate::model_validation::models::{ModelValidationFacts, ProjectValidationFacts}; +use crate::model_validation::types::Check; +use crate::types::AuthoredNode; + +/// Accept only what every Python validator accepts; a rejection means Python must decide. +pub fn accept_model_config( + entries: Vec<(N, N)>, + project: &ProjectValidationFacts, + facts: &ModelValidationFacts<'_>, +) -> Check { + let config = ConfigView::new(entries); + check_references(facts.references, project)?; + check_incremental(&config, facts)?; + check_project_capability(&config, project)?; + check_contract_and_incremental_keys(&config)?; + check_snapshot(&config, facts)?; + check_custom_materialization(&config, project)?; + check_storage_policies(&config, facts)?; + check_migration(&config, facts)?; + check_placeholders(&config, project, facts) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs new file mode 100644 index 0000000000..bd1459bc69 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs @@ -0,0 +1,3 @@ +pub mod accept_model_config; +pub mod retention_override; +pub mod table_type_override; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs new file mode 100644 index 0000000000..a6f3222f9d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs @@ -0,0 +1,27 @@ +//! Read a MODEL header's `time_travel_retention` as `resolve_time_travel_retention` does. + +use crate::model_validation::_helpers::config::optional_text; +use crate::model_validation::_helpers::durations::parse_duration; +use crate::model_validation::constants::{DISABLED_RETENTION, INHERIT_POLICY, ZERO_DAY_DURATION}; +use crate::model_validation::models::{Rejected, RetentionOverride}; +use crate::types::AuthoredNode; + +/// Return the header retention override, or reject a value Python refuses. +pub fn retention_override( + value: Option<&N>, +) -> Result { + let Some(text) = optional_text(value)? else { + return Ok(RetentionOverride::Inherit); + }; + match text.as_str() { + INHERIT_POLICY => Ok(RetentionOverride::Inherit), + DISABLED_RETENTION => Ok(RetentionOverride::Unmanaged), + ZERO_DAY_DURATION => Ok(RetentionOverride::Days(0)), + _ => match parse_duration(&text)? { + Some(duration) if duration.is_whole_days() => u64::try_from(duration.days()) + .map(RetentionOverride::Days) + .map_err(|_| Rejected), + _ => Err(Rejected), + }, + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs new file mode 100644 index 0000000000..daed7e16e8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/table_type_override.rs @@ -0,0 +1,20 @@ +//! Read a MODEL header's `table_type` as `resolve_table_type` does. + +use crate::model_validation::_helpers::config::optional_text; +use crate::model_validation::constants::{ + INHERIT_POLICY, PERMANENT_TABLE_TYPE, TRANSIENT_TABLE_TYPE, +}; +use crate::model_validation::models::{Rejected, TableTypeOverride}; +use crate::types::AuthoredNode; + +/// Return the header table-type override, or reject a value Python refuses. +pub fn table_type_override( + value: Option<&N>, +) -> Result { + match optional_text(value)?.as_deref() { + None | Some(INHERIT_POLICY) => Ok(TableTypeOverride::Inherit), + Some(PERMANENT_TABLE_TYPE) => Ok(TableTypeOverride::Permanent), + Some(TRANSIENT_TABLE_TYPE) => Ok(TableTypeOverride::Transient), + Some(_) => Err(Rejected), + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/mod.rs b/crates/sqlbuild-model-config/src/model_validation/mod.rs new file mode 100644 index 0000000000..a028155050 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/model_validation/models.rs b/crates/sqlbuild-model-config/src/model_validation/models.rs new file mode 100644 index 0000000000..342c566bbc --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/models.rs @@ -0,0 +1,69 @@ +//! What the model validators read besides the effective config values. + +use std::collections::HashSet; + +/// Project-wide facts every model's validation reads. +#[derive(Clone, Debug, Default)] +pub struct ProjectValidationFacts { + /// Discovered custom materialization names. + pub custom_materializations: HashSet, + /// Whether the project enables concurrent microbatches. + pub microbatch_concurrency: bool, + /// Discovered model names. + pub models: HashSet, + /// Discovered seed names. + pub seeds: HashSet, + /// Discovered source names. + pub sources: HashSet, + /// Discovered SQL function names, scalar and table. + pub functions: HashSet, + /// Discovered table function names. + pub table_functions: HashSet, +} + +/// One extracted SQL reference: its `SqlReferenceKind` value and name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelReference { + pub kind: String, + pub name: String, +} + +/// One model's facts beyond its config values. +#[derive(Clone, Debug)] +pub struct ModelValidationFacts<'a> { + pub model_name: &'a str, + /// Merged references in extraction order. + pub references: &'a [ModelReference], + /// Column names of the model's declared schema, when one applies. + pub declared_columns: Option<&'a [String]>, + /// The expanded query SQL. + pub query_sql: &'a str, + /// Whether the resolved time-travel retention is unmanaged. + pub retention_unmanaged: bool, + /// Whether the resolved table type was declared by any layer. + pub table_type_declared: bool, +} + +/// Why native validation cannot accept a config: Python must run and decide. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Rejected; + +/// The header's `time_travel_retention`, applied over the inherited policy. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RetentionOverride { + /// Absent or `inherit`: keep the inherited policy. + Inherit, + /// `disabled`: retention is unmanaged. + Unmanaged, + /// A whole-day duration, or `0d`. + Days(u64), +} + +/// The header's `table_type`, applied over the inherited type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableTypeOverride { + /// Absent or `inherit`: keep the inherited type. + Inherit, + Permanent, + Transient, +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests.rs b/crates/sqlbuild-model-config/src/model_validation/tests.rs new file mode 100644 index 0000000000..eac226ccf3 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_accept_model_config.rs"] +mod accept_model_config; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_storage_overrides.rs"] +mod storage_overrides; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs new file mode 100644 index 0000000000..036cd9cd19 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -0,0 +1,61 @@ +use crate::model_validation::models::ProjectValidationFacts; +use crate::tests::test_types::Value; + +/// A list of strings. +pub(super) fn strings(values: &[&str]) -> Value { + Value::List(values.iter().map(|value| Value::Str(leak(value))).collect()) +} + +/// A string that lives for the whole test run. +fn leak(value: &str) -> &'static str { + Box::leak(value.to_owned().into_boxed_str()) +} + +/// A mapping from string keys. +pub(super) fn map(entries: Vec<(&'static str, Value)>) -> Value { + Value::Map( + entries + .into_iter() + .map(|(key, value)| (Value::Str(key), value)) + .collect(), + ) +} + +/// The project's resources and custom materializations. +pub(super) fn project() -> ProjectValidationFacts { + let names = |values: &[&str]| values.iter().map(|value| (*value).to_owned()).collect(); + ProjectValidationFacts { + custom_materializations: names(&["ledger"]), + microbatch_concurrency: false, + models: names(&["orders", "customers"]), + seeds: names(&["regions"]), + sources: names(&["raw.orders"]), + functions: names(&["cents", "order_lines"]), + table_functions: names(&["order_lines"]), + } +} + +/// An append incremental config with a daily timestamp cursor, plus `extra`. +pub(super) fn incremental(extra: Vec<(&'static str, Value)>) -> Vec<(&'static str, Value)> { + let mut config = vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("updated_at")), + ("cursor_type", Value::Str("timestamp")), + ("cursor_grain", Value::Str("day")), + ]; + config.extend(extra); + config +} + +/// A timestamp snapshot config, plus `extra`. +pub(super) fn snapshot(extra: Vec<(&'static str, Value)>) -> Vec<(&'static str, Value)> { + let mut config = vec![ + ("materialized", Value::Str("snapshot")), + ("unique_key", Value::Str("order_id")), + ("snapshot_strategy", Value::Str("timestamp")), + ("updated_at", Value::Str("updated_at")), + ]; + config.extend(extra); + config +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_accept_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_accept_model_config.rs new file mode 100644 index 0000000000..981b501074 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_accept_model_config.rs @@ -0,0 +1,322 @@ +use crate::model_validation::main::accept_model_config::accept_model_config; +use crate::model_validation::models::{ModelReference, ModelValidationFacts}; +use crate::model_validation::tests::helpers::{incremental, map, project, snapshot, strings}; +use crate::model_validation::tests::test_types::AcceptTestCase; +use crate::tests::test_types::Value; + +#[test] +fn given_effective_configs_when_validating_natively_then_only_python_valid_configs_pass() { + let test_cases = [ + AcceptTestCase { + description: "a plain table with known references", + config: vec![("materialized", Value::Str("table"))], + references: &[ + "ref:orders", + "seed:regions", + "source:raw.orders", + "udf:cents", + ], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "an unknown model reference", + config: vec![("materialized", Value::Str("table"))], + references: &["ref:invoices"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a table function called as a scalar function", + config: vec![], + references: &["udf:order_lines"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a dbt reference always defers to Python", + config: vec![], + references: &["dbt_ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "an append incremental with ordered ISO bounds", + config: incremental(vec![ + ("cursor_start", Value::Str("2024-01-01")), + ("cursor_end", Value::Str("2024-02-01T00:00:00+00:00")), + ("append_cursor_inclusive", Value::Bool(true)), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "cursor bounds out of order", + config: incremental(vec![ + ("cursor_start", Value::Str("2024-02-01")), + ("cursor_end", Value::Str("2024-01-01")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a start bound shifted before year 1 in UTC defers", + config: incremental(vec![ + ("cursor_start", Value::Str("0001-01-01T00:00:00+01:00")), + ("cursor_end", Value::Str("2024-01-01")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "an end bound shifted past year 9999 in UTC defers", + config: incremental(vec![ + ("cursor_start", Value::Str("2024-01-01")), + ("cursor_end", Value::Str("9999-12-31T23:59:00-01:00")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "bounds at the edges of years 1 and 9999 in UTC", + config: incremental(vec![ + ("cursor_start", Value::Str("0001-01-01T01:00:00+01:00")), + ("cursor_end", Value::Str("9999-12-31T22:59:00-01:00")), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "an unusual ISO timestamp defers", + config: incremental(vec![("cursor_start", Value::Str("2024-W01-1"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a timestamp cursor without a grain", + config: vec![ + ("materialized", Value::Str("incremental")), + ("incremental_strategy", Value::Str("append")), + ("cursor", Value::Str("updated_at")), + ("cursor_type", Value::Str("timestamp")), + ], + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a watermark microbatch with a satisfiable limit", + config: incremental(vec![ + ("incremental_strategy", Value::Str("delete_insert")), + ("incremental_mode", Value::Str("microbatch")), + ("microbatch_strategy", Value::Str("watermark")), + ("cursor_watermark_mode", Value::Str("all")), + ("batch_size", Value::Str("1d")), + ("lookback", Value::Str("3d")), + ( + "cursor_inputs", + map(vec![( + "orders", + map(vec![ + ("column", Value::Str("updated_at")), + ("roles", strings(&["filter", "watermark"])), + ]), + )]), + ), + ( + "microbatch_limit", + map(vec![ + ("max_batches", Value::Int(5)), + ("action", Value::Str("cap_from_start")), + ]), + ), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "a watermark limit below the lookback requirement", + config: incremental(vec![ + ("incremental_strategy", Value::Str("delete_insert")), + ("incremental_mode", Value::Str("microbatch")), + ("microbatch_strategy", Value::Str("watermark")), + ("cursor_watermark_mode", Value::Str("all")), + ("batch_size", Value::Str("1d")), + ("lookback", Value::Str("3d")), + ( + "cursor_inputs", + map(vec![( + "orders", + map(vec![ + ("column", Value::Str("updated_at")), + ("roles", strings(&["watermark"])), + ]), + )]), + ), + ("max_microbatches", Value::Int(3)), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "merge exclusions overlapping the unique key", + config: incremental(vec![ + ("incremental_strategy", Value::Str("merge")), + ("unique_key", Value::Str("Order_Id")), + ("merge_exclude_columns", strings(&["order_id"])), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a bounded replay policy", + config: incremental(vec![("replay_on_change", Value::Str("bounded- 14d"))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "an incremental-only key on a table", + config: vec![ + ("materialized", Value::Str("table")), + ("full_refresh", Value::Bool(true)), + ], + references: &[], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a timestamp snapshot with observed history", + config: snapshot(vec![ + ("observed_at", Value::Str("loaded_at")), + ("historical_input", Value::Str("snapshot")), + ("invalidate_hard_deletes", Value::Bool(true)), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "a check snapshot mixing the wildcard", + config: snapshot(vec![ + ("snapshot_strategy", Value::Str("check")), + ("check_columns", strings(&["*", "status"])), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "an enforced contract naming an undeclared unique key", + config: snapshot(vec![ + ("contract", Value::Str("enforced")), + ("columns", map(vec![("updated_at", Value::Null)])), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a custom materialization with matching placeholders", + config: vec![ + ("materialized", Value::Str("ledger")), + ("placeholders", map(vec![("region", Value::Str("emea"))])), + ], + references: &[], + query_sql: "select @@@region, '@@@' as marker", + expected_accepted: true, + }, + AcceptTestCase { + description: "placeholders on a built-in materialization", + config: vec![("materialized", Value::Str("view"))], + references: &[], + query_sql: "select @@@region", + expected_accepted: false, + }, + AcceptTestCase { + description: "an unknown custom materialization", + config: vec![("materialized", Value::Str("archive"))], + references: &[], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "a table migrating from an old name with an old-name view", + config: vec![ + ("materialized", Value::Str("table")), + ("migrate_from", Value::Str(" legacy_orders ")), + ("old_name_view", Value::Str("7d")), + ], + references: &[], + query_sql: "select 1", + expected_accepted: true, + }, + AcceptTestCase { + description: "a view forcing a migration", + config: vec![ + ("materialized", Value::Str("view")), + ("migrate_from", Value::Str("legacy_orders")), + ("migrate_force", Value::Bool(true)), + ], + references: &[], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "an old-name view without a duration", + config: vec![("old_name_view", Value::Null)], + references: &[], + query_sql: "select 1", + expected_accepted: false, + }, + AcceptTestCase { + description: "concurrent batches without the project capability", + config: incremental(vec![("batch_concurrency", Value::Int(2))]), + references: &["ref:orders"], + query_sql: "select 1", + expected_accepted: false, + }, + ]; + + for test_case in test_cases { + let references: Vec = test_case + .references + .iter() + .filter_map(|reference| reference.split_once(':')) + .map(|(kind, name)| ModelReference { + kind: kind.to_owned(), + name: name.to_owned(), + }) + .collect(); + let facts = ModelValidationFacts { + model_name: "orders_daily", + references: &references, + declared_columns: None, + query_sql: test_case.query_sql, + retention_unmanaged: true, + table_type_declared: false, + }; + let entries = test_case + .config + .into_iter() + .map(|(key, value)| (Value::Str(key), value)) + .collect(); + + let accepted = accept_model_config(entries, &project(), &facts) == Ok(()); + + assert_eq!( + accepted, test_case.expected_accepted, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs new file mode 100644 index 0000000000..5e780d1315 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_storage_overrides.rs @@ -0,0 +1,99 @@ +use crate::model_validation::main::retention_override::retention_override; +use crate::model_validation::main::table_type_override::table_type_override; +use crate::model_validation::models::{Rejected, RetentionOverride, TableTypeOverride}; +use crate::model_validation::tests::test_types::{ + RetentionOverrideTestCase, TableTypeOverrideTestCase, +}; +use crate::tests::test_types::Value; + +#[test] +fn given_header_retention_values_when_reading_then_python_policies_result() { + let test_cases = [ + RetentionOverrideTestCase { + description: "absent", + value: None, + expected_override: Ok(RetentionOverride::Inherit), + }, + RetentionOverrideTestCase { + description: "inherit", + value: Some(Value::Str("inherit")), + expected_override: Ok(RetentionOverride::Inherit), + }, + RetentionOverrideTestCase { + description: "disabled", + value: Some(Value::Str("disabled")), + expected_override: Ok(RetentionOverride::Unmanaged), + }, + RetentionOverrideTestCase { + description: "zero days", + value: Some(Value::Str("0d")), + expected_override: Ok(RetentionOverride::Days(0)), + }, + RetentionOverrideTestCase { + description: "whole days", + value: Some(Value::Str("7d")), + expected_override: Ok(RetentionOverride::Days(7)), + }, + RetentionOverrideTestCase { + description: "hours are not whole days", + value: Some(Value::Str("1d6h")), + expected_override: Err(Rejected), + }, + RetentionOverrideTestCase { + description: "not a string", + value: Some(Value::Int(7)), + expected_override: Err(Rejected), + }, + ]; + + for test_case in test_cases { + let resolved = retention_override(test_case.value.as_ref()); + + assert_eq!( + resolved, test_case.expected_override, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_header_table_types_when_reading_then_python_types_result() { + let test_cases = [ + TableTypeOverrideTestCase { + description: "absent", + value: None, + expected_override: Ok(TableTypeOverride::Inherit), + }, + TableTypeOverrideTestCase { + description: "inherit", + value: Some(Value::Str("inherit")), + expected_override: Ok(TableTypeOverride::Inherit), + }, + TableTypeOverrideTestCase { + description: "permanent", + value: Some(Value::Str("permanent")), + expected_override: Ok(TableTypeOverride::Permanent), + }, + TableTypeOverrideTestCase { + description: "transient", + value: Some(Value::Str("transient")), + expected_override: Ok(TableTypeOverride::Transient), + }, + TableTypeOverrideTestCase { + description: "unknown", + value: Some(Value::Str("temporary")), + expected_override: Err(Rejected), + }, + ]; + + for test_case in test_cases { + let resolved = table_type_override(test_case.value.as_ref()); + + assert_eq!( + resolved, test_case.expected_override, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs new file mode 100644 index 0000000000..1f962f3aa8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs @@ -0,0 +1,26 @@ +use crate::model_validation::models::{Rejected, RetentionOverride, TableTypeOverride}; +use crate::tests::test_types::Value; + +/// One effective config, its references, and whether native validation accepts it. +pub(super) struct AcceptTestCase { + pub(super) description: &'static str, + pub(super) config: Vec<(&'static str, Value)>, + /// `kind:name` per reference. + pub(super) references: &'static [&'static str], + pub(super) query_sql: &'static str, + pub(super) expected_accepted: bool, +} + +/// One header `time_travel_retention` value and the override Python resolves. +pub(super) struct RetentionOverrideTestCase { + pub(super) description: &'static str, + pub(super) value: Option, + pub(super) expected_override: Result, +} + +/// One header `table_type` value and the override Python resolves. +pub(super) struct TableTypeOverrideTestCase { + pub(super) description: &'static str, + pub(super) value: Option, + pub(super) expected_override: Result, +} diff --git a/crates/sqlbuild-model-config/src/model_validation/types.rs b/crates/sqlbuild-model-config/src/model_validation/types.rs new file mode 100644 index 0000000000..a0c3be4a76 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/types.rs @@ -0,0 +1,6 @@ +//! The outcome type native validation threads through every check. + +use crate::model_validation::models::Rejected; + +/// Native validation's outcome; any rejection re-runs the Python validators. +pub type Check = Result<(), Rejected>; diff --git a/crates/sqlbuild-model-config/src/path_defaults/_helpers/matching.rs b/crates/sqlbuild-model-config/src/path_defaults/_helpers/matching.rs new file mode 100644 index 0000000000..fe4ffcfb18 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/_helpers/matching.rs @@ -0,0 +1,57 @@ +//! Path-default glob matching and specificity, as `path_defaults/_helpers/matching.py`. + +use crate::path_defaults::constants::{RECURSIVE_SEGMENT_GLOB, SINGLE_SEGMENT_GLOB}; + +/// Return whether a path-default pattern matches a prefix of the model path's segments. +pub(crate) fn matches_prefix(path_parts: &[&str], key: &str) -> bool { + let pattern: Vec<&str> = key.split('/').collect(); + let mut table = vec![vec![false; path_parts.len() + 1]; pattern.len() + 1]; + table[pattern.len()].fill(true); + for pattern_index in (0..pattern.len()).rev() { + let part = pattern[pattern_index]; + for path_index in (0..=path_parts.len()).rev() { + table[pattern_index][path_index] = if part == RECURSIVE_SEGMENT_GLOB { + table[pattern_index + 1][path_index] + || (path_index < path_parts.len() && table[pattern_index][path_index + 1]) + } else { + path_index < path_parts.len() + && (part == SINGLE_SEGMENT_GLOB || part == path_parts[path_index]) + && table[pattern_index + 1][path_index + 1] + }; + } + } + table[0][0] +} + +fn is_glob(part: &str) -> bool { + part == SINGLE_SEGMENT_GLOB || part == RECURSIVE_SEGMENT_GLOB +} + +/// Return whether a key holds a `*` or `**` segment. +pub(crate) fn is_wildcard(key: &str) -> bool { + key.split('/').any(is_glob) +} + +/// Return the number of `/`-separated segments in a key. +pub(crate) fn segment_count(key: &str) -> usize { + key.split('/').count() +} + +/// Return the literal, single-glob and negated recursive-glob segment counts. +pub(crate) fn specificity(key: &str) -> (usize, usize, isize) { + let parts: Vec<&str> = key.split('/').collect(); + let literal = parts.iter().filter(|part| !is_glob(part)).count(); + let single = parts + .iter() + .filter(|part| **part == SINGLE_SEGMENT_GLOB) + .count(); + let recursive = parts + .iter() + .filter(|part| **part == RECURSIVE_SEGMENT_GLOB) + .count(); + ( + literal, + single, + -isize::try_from(recursive).unwrap_or(isize::MAX), + ) +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/_helpers/mod.rs b/crates/sqlbuild-model-config/src/path_defaults/_helpers/mod.rs new file mode 100644 index 0000000000..87a72823fc --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod matching; diff --git a/crates/sqlbuild-model-config/src/path_defaults/constants.rs b/crates/sqlbuild-model-config/src/path_defaults/constants.rs new file mode 100644 index 0000000000..3ae717449b --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/constants.rs @@ -0,0 +1,8 @@ +//! Path-default glob segments. + +/// A segment matching exactly one path segment. +pub const SINGLE_SEGMENT_GLOB: &str = "*"; +/// A segment matching any number of path segments. +pub const RECURSIVE_SEGMENT_GLOB: &str = "**"; +/// The project-relative folder prefix model paths drop before matching. +pub const MODELS_PREFIX: &str = "models/"; diff --git a/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs b/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs new file mode 100644 index 0000000000..ac0e1e3b10 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs @@ -0,0 +1 @@ +pub mod select_path_default; diff --git a/crates/sqlbuild-model-config/src/path_defaults/main/select_path_default.rs b/crates/sqlbuild-model-config/src/path_defaults/main/select_path_default.rs new file mode 100644 index 0000000000..c0ec644032 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/main/select_path_default.rs @@ -0,0 +1,42 @@ +//! `select_path_default`: the nearest path-default key for a model path. + +use crate::path_defaults::_helpers::matching::{ + is_wildcard, matches_prefix, segment_count, specificity, +}; +use crate::path_defaults::constants::MODELS_PREFIX; +use crate::path_defaults::models::PathDefaultChoice; + +/// Select the path default for a project-relative model path as Python does. +pub fn select_path_default(model_path: &str, keys: &[String]) -> PathDefaultChoice { + let normalized = model_path.replace('\\', "/"); + let normalized = normalized + .strip_prefix(MODELS_PREFIX) + .unwrap_or(&normalized); + let path_parts: Vec<&str> = normalized + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + let mut matched: Vec<&String> = keys + .iter() + .filter(|key| matches_prefix(&path_parts, key)) + .collect(); + matched.sort(); + let mut literal = matched.iter().filter(|key| !is_wildcard(key)).peekable(); + if literal.peek().is_some() { + let mut best: Option<&String> = None; + for key in literal { + if best.is_none_or(|current| segment_count(key) > segment_count(current)) { + best = Some(key); + } + } + return PathDefaultChoice::Selected(best.cloned()); + } + let Some(best_score) = matched.iter().map(|key| specificity(key)).max() else { + return PathDefaultChoice::Selected(None); + }; + let mut best = matched.iter().filter(|key| specificity(key) == best_score); + match (best.next(), best.next()) { + (Some(key), None) => PathDefaultChoice::Selected(Some((*key).clone())), + _ => PathDefaultChoice::Conflict, + } +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/mod.rs b/crates/sqlbuild-model-config/src/path_defaults/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/path_defaults/models.rs b/crates/sqlbuild-model-config/src/path_defaults/models.rs new file mode 100644 index 0000000000..f7f90b4a0a --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/models.rs @@ -0,0 +1,10 @@ +//! The outcome of selecting one model's path default. + +/// The selected path-default key, or a conflict Python reports. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PathDefaultChoice { + /// The nearest matching key, or `None` when no key matches. + Selected(Option), + /// Several equally specific wildcard keys match. + Conflict, +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/tests.rs b/crates/sqlbuild-model-config/src/path_defaults/tests.rs new file mode 100644 index 0000000000..892f61f6d0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_select_path_default.rs"] +mod select_path_default; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs new file mode 100644 index 0000000000..67eae9897b --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs @@ -0,0 +1,57 @@ +use crate::path_defaults::main::select_path_default::select_path_default; +use crate::path_defaults::models::PathDefaultChoice; +use crate::path_defaults::tests::test_types::SelectTestCase; + +#[test] +fn given_path_default_keys_when_selecting_then_the_python_choice_is_made() { + let test_cases = [ + SelectTestCase { + description: "the deepest literal prefix wins", + model_path: "models/staging/orders/orders.sql", + keys: &["staging", "staging/orders", "*/orders"], + expected_choice: PathDefaultChoice::Selected(Some("staging/orders".to_owned())), + }, + SelectTestCase { + description: "equally deep literal keys keep the first in sorted order", + model_path: "staging/orders.sql", + keys: &["staging/orders.sql", "staging/aaaaa"], + expected_choice: PathDefaultChoice::Selected(Some("staging/orders.sql".to_owned())), + }, + SelectTestCase { + description: "the most literal wildcard wins", + model_path: "models\\marts\\finance\\revenue.sql", + keys: &["**/finance", "marts/*", "marts/**/finance"], + expected_choice: PathDefaultChoice::Selected(Some("marts/**/finance".to_owned())), + }, + SelectTestCase { + description: "a recursive glob matches zero segments", + model_path: "marts/revenue.sql", + keys: &["marts/**"], + expected_choice: PathDefaultChoice::Selected(Some("marts/**".to_owned())), + }, + SelectTestCase { + description: "equally specific wildcards conflict", + model_path: "marts/finance/revenue.sql", + keys: &["*/finance", "marts/*"], + expected_choice: PathDefaultChoice::Conflict, + }, + SelectTestCase { + description: "no key matches", + model_path: "staging/orders.sql", + keys: &["marts", "*/finance"], + expected_choice: PathDefaultChoice::Selected(None), + }, + ]; + + for test_case in test_cases { + let keys: Vec = test_case.keys.iter().map(|key| (*key).to_owned()).collect(); + + let choice = select_path_default(test_case.model_path, &keys); + + assert_eq!( + choice, test_case.expected_choice, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/tests/test_types.rs b/crates/sqlbuild-model-config/src/path_defaults/tests/test_types.rs new file mode 100644 index 0000000000..041df94692 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/tests/test_types.rs @@ -0,0 +1,9 @@ +use crate::path_defaults::models::PathDefaultChoice; + +/// One model path, the configured keys, and the selection Python makes. +pub(super) struct SelectTestCase { + pub(super) description: &'static str, + pub(super) model_path: &'static str, + pub(super) keys: &'static [&'static str], + pub(super) expected_choice: PathDefaultChoice, +} diff --git a/crates/sqlbuild-model-config/src/tests/test_types.rs b/crates/sqlbuild-model-config/src/tests/test_types.rs index b524b6ee45..ba3ab491e4 100644 --- a/crates/sqlbuild-model-config/src/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/tests/test_types.rs @@ -29,6 +29,13 @@ impl AuthoredNode for Value { } } + fn integer(&self) -> Option { + match self { + Self::Int(value) => Some(*value), + _ => None, + } + } + fn text(&self) -> Option { match self { Self::Str(text) => Some((*text).to_owned()), diff --git a/crates/sqlbuild-model-config/src/types.rs b/crates/sqlbuild-model-config/src/types.rs index a8f798c4bf..bc9fbe7318 100644 --- a/crates/sqlbuild-model-config/src/types.rs +++ b/crates/sqlbuild-model-config/src/types.rs @@ -21,6 +21,8 @@ pub enum NodeKind { pub trait AuthoredNode: Clone { /// Return this value's kind. fn kind(&self) -> NodeKind; + /// Return an integer that is not a boolean and fits in `i64`, or `None`. + fn integer(&self) -> Option; /// Return the text of a string value, or `None` when it is not readable UTF-8 text. fn text(&self) -> Option; /// Return whether this is a string value equal to `text`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs index b8edbf6e2c..ebec35d8f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/authored_nodes.rs @@ -35,6 +35,17 @@ impl AuthoredNode for PyNode<'_> { } } + fn integer(&self) -> Option { + if !self.0.is_instance_of::() + && self.0.is_instance_of::() + && let Ok(number) = self.0.extract::() + { + Some(number) + } else { + None + } + } + fn text(&self) -> Option { self.with_text(str::to_owned) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 9a2a60574d..6a6242a625 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -21,22 +21,42 @@ const INVALID_OUTCOME: &str = "invalid"; const UNSUPPORTED_OUTCOME: &str = "unsupported"; /// Variables, the process environment and context values, all read through Python. -struct PythonHost<'py> { +pub(crate) struct PythonHost<'py> { py: Python<'py>, sources: TemplateSources<'py>, reads: RefCell>, } +impl<'py> PythonHost<'py> { + /// Read variables, the environment and context values from these Python objects. + pub(crate) fn new(py: Python<'py>, sources: TemplateSources<'py>) -> Self { + Self { + py, + sources, + reads: RefCell::new(Vec::new()), + } + } + + /// Return the `(kind, name)` reads in order. + pub(crate) fn into_reads(self) -> Vec<(&'static str, String)> { + self.reads.into_inner() + } +} + /// The variables, the `os.environ` mapping and the context values one expansion reads. #[derive(FromPyObject)] -struct TemplateSources<'py>(Bound<'py, PyDict>, Bound<'py, PyAny>, Bound<'py, PyDict>); +pub(crate) struct TemplateSources<'py>( + pub(crate) Bound<'py, PyDict>, + pub(crate) Bound<'py, PyAny>, + pub(crate) Bound<'py, PyDict>, +); /// `allow_context`, `preserve_context_tokens` and `preserve_unknown_context`. #[derive(FromPyObject)] struct TemplateFlags(bool, bool, bool); /// A template failure, or a Python error raised while building the expanded containers. -enum Stop { +pub(crate) enum Stop { Failure(TemplateFailure), Python(PyErr), } @@ -126,13 +146,9 @@ fn expand_config_templates<'py>( preserve_context_tokens, preserve_unknown_context, }; - let host = PythonHost { - py, - sources, - reads: RefCell::new(Vec::new()), - }; + let host = PythonHost::new(py, sources); match expanded(&host, &value, options) { - Ok(result) => Ok((result, host.reads.into_inner()) + Ok(result) => Ok((result, host.into_reads()) .into_pyobject(py)? .into_any() .unbind()), @@ -146,7 +162,7 @@ fn expand_config_templates<'py>( } } -fn expanded<'py>( +pub(crate) fn expanded<'py>( host: &PythonHost<'py>, value: &Bound<'py, PyAny>, options: TemplateOptions, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 464ecd1be5..aef698874b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -2,4 +2,6 @@ pub(crate) mod authored_nodes; pub(crate) mod config_presence; pub(crate) mod config_templates; pub(crate) mod header_metadata; +pub(crate) mod model_config_builder; +pub(crate) mod model_validator; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs new file mode 100644 index 0000000000..74310c02e2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -0,0 +1,694 @@ +//! Effective model config layered and templated natively, as `build_model_config` builds it. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{IntoPyObject, PyErr, pyclass, pymethods}; +use sqlbuild_model_config::config_presence::main::contains_macro_call::contains_macro_call; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::models::Presence; +use sqlbuild_model_config::model_validation::main::retention_override::retention_override; +use sqlbuild_model_config::model_validation::main::table_type_override::table_type_override; +use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; +use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; +use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; +use sqlbuild_model_config::templates::models::{TemplateFailure, TemplateOptions}; +use sqlbuild_model_config::types::AuthoredNode; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_templates::{ + PythonHost, Stop, TemplateSources, expanded, +}; + +const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const TAGS_KEY: &str = "tags"; +const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; +const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; +const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; +const ALIAS_KEY: &str = "alias"; +const MATERIALIZED_KEY: &str = "materialized"; +const FULL_REFRESH_KEY: &str = "full_refresh"; +const RETENTION_KEY: &str = "time_travel_retention"; +const TABLE_TYPE_KEY: &str = "table_type"; +const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +const PRESERVE_TARGET_VALUE: &str = "preserve"; +const RUN_ID_CONTEXT: &str = "run.id"; +const RUN_TARGET_CONTEXT: &str = "run.target"; +const MODEL_NAME_CONTEXT: &str = "model.name"; +const MODEL_DATABASE_CONTEXT: &str = "model.database"; +const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; +const MODEL_ALIAS_CONTEXT: &str = "model.alias"; +const DESTINATION_DATABASE_CONTEXT: &str = "destination.database"; +const DESTINATION_SCHEMA_CONTEXT: &str = "destination.schema"; +const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; +const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; + +/// Why the native build stops: Python builds this model's config, or a Python error. +enum Halt { + Defer, + Error(PyErr), +} + +impl From for Halt { + fn from(error: PyErr) -> Self { + Self::Error(error) + } +} + +impl From for Halt { + fn from(stop: Stop) -> Self { + match stop { + Stop::Failure(_) => Self::Defer, + Stop::Python(error) => Self::Error(error), + } + } +} + +impl From for Halt { + fn from(_: TemplateFailure) -> Self { + Self::Defer + } +} + +type Built = Result; + +/// The project-wide inputs every model's config build reads, captured once per compile. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeModelConfigBuilder { + defaults: Py, + path_defaults: Py, + path_keys: Vec, + hook_entry_types: Py, + variables: Py, + environment: Py, + target_name: Option>, + run_id: Py, + target_namespace: Option<(Py, Py)>, +} + +/// One model's built config before Python wraps it in `CompileModelConfig`. +struct BuiltConfig<'py> { + values: Bound<'py, PyDict>, + header_keys: Vec, + logical_schema: Option>, + layer_schema_configured: bool, + logical_database: Option>, + retention: RetentionOverride, + table_type: TableTypeOverride, + reads: Vec<(&'static str, String)>, +} + +#[pymethods] +impl NativeModelConfigBuilder { + #[new] + fn new( + layers: (Bound<'_, PyDict>, Bound<'_, PyDict>, Bound<'_, PyTuple>), + sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), + run: (Bound<'_, PyAny>, Bound<'_, PyAny>), + target_namespace: Option<(Bound<'_, PyAny>, Bound<'_, PyAny>)>, + ) -> PyResult { + let (defaults, path_defaults, hook_entry_types) = layers; + let path_keys = path_defaults + .keys() + .iter() + .map(|key| key.extract::()) + .collect::>>()?; + let (target_name, run_id) = run; + Ok(Self { + defaults: defaults.unbind(), + path_defaults: path_defaults.unbind(), + path_keys, + hook_entry_types: hook_entry_types.unbind(), + variables: sources.0.unbind(), + environment: sources.1.unbind(), + target_name: (!target_name.is_none()).then(|| target_name.unbind()), + run_id: run_id.unbind(), + target_namespace: target_namespace + .map(|(database, schema)| (database.unbind(), schema.unbind())), + }) + } + + /// Return `(True, key)` for the selected path default, or `(False, None)` on a conflict. + fn path_default(&self, model_path: &str) -> PyResult<(bool, Option)> { + compiler_guard(|| { + Ok(match select_path_default(model_path, &self.path_keys) { + PathDefaultChoice::Selected(key) => (true, key), + PathDefaultChoice::Conflict => (false, None), + }) + }) + } + + /// Return the built config parts, or `None` when Python must build this model's config. + fn build<'py>( + &self, + py: Python<'py>, + header: Bound<'py, PyDict>, + matched_path_default: Option<&str>, + model_name: &str, + ) -> PyResult>> { + compiler_guard( + || match self.build_config(py, &header, matched_path_default, model_name) { + Ok(built) => built_tuple(py, built).map(Some), + Err(Halt::Defer) => Ok(None), + Err(Halt::Error(error)) => Err(error), + }, + ) + } +} + +impl NativeModelConfigBuilder { + fn build_config<'py>( + &self, + py: Python<'py>, + header: &Bound<'py, PyDict>, + matched_path_default: Option<&str>, + model_name: &str, + ) -> Built> { + check_header_tags(header)?; + let path_values = matched_path_default + .map(|key| self.path_default_values(py, key)) + .transpose()?; + let mut layered = self.project_defaults(py)?; + if let Some(path_values) = &path_values { + layered = merged_layer(py, &layered, path_values)?; + } + let layered = merged_layer(py, &layered, header)?; + self.check_hooks(py, &layered)?; + let mut hooks: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + for key in HOOK_KEYS { + if let Some(value) = layered.get_item(key)? { + hooks.push((key, value)); + layered.del_item(key)?; + } + } + let has_templates = present(contains_template(&PyNode(layered.clone().into_any())))?; + let mut expansion = Expansion { + builder: self, + py, + model_name, + reads: Vec::new(), + }; + let model_values = if has_templates { + let early = expansion.expand( + &layered, + &self.variables.bind(py).clone(), + expansion.run_context()?, + true, + )?; + let empty = PyDict::new(py); + let first = expansion.expand( + &early, + &empty, + expansion.model_context(&early, false)?, + true, + )?; + expansion.expand( + &first, + &empty, + expansion.model_context(&first, false)?, + true, + )? + } else { + layered + }; + let logical_schema = string_entry(&model_values, SCHEMA_KEY)?; + let logical_database = string_entry(&model_values, DATABASE_KEY)?; + let layer_schema_configured = header.contains(SCHEMA_KEY)? + || path_values + .as_ref() + .map(|values| values.contains(SCHEMA_KEY)) + .transpose()? + .unwrap_or(false); + let namespaced = + expansion.namespaced_values(&model_values, &logical_database, &logical_schema)?; + let needs_target_templates = has_templates + || [DATABASE_KEY, SCHEMA_KEY] + .iter() + .map(|key| entry_has_template(&namespaced, key)) + .collect::>>()? + .into_iter() + .any(|found| found); + let resolved = if needs_target_templates { + let empty = PyDict::new(py); + expansion.expand( + &namespaced, + &empty, + expansion.model_context(&namespaced, true)?, + false, + )? + } else { + namespaced + }; + for (key, value) in hooks { + resolved.set_item(key, value)?; + } + let values = storage_free_values(py, &resolved, header)?; + let retention = retention_override(header.get_item(RETENTION_KEY)?.map(PyNode).as_ref()) + .map_err(|_| Halt::Defer)?; + let table_type = table_type_override(header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref()) + .map_err(|_| Halt::Defer)?; + check_no_config_macros(py, &values)?; + Ok(BuiltConfig { + values, + header_keys: sorted_keys(header)?, + logical_schema, + layer_schema_configured, + logical_database, + retention, + table_type, + reads: expansion.reads, + }) + } + + fn path_default_values<'py>(&self, py: Python<'py>, key: &str) -> Built> { + let values = self + .path_defaults + .bind(py) + .get_item(key)? + .ok_or(Halt::Defer)?; + values.downcast_into::().map_err(|_| Halt::Defer) + } + + /// Return a fresh `project_defaults_to_mapping` result, with its own `tags` list. + fn project_defaults<'py>(&self, py: Python<'py>) -> Built> { + let defaults = self.defaults.bind(py).copy()?; + if let Some(tags) = defaults.get_item(TAGS_KEY)? { + let tags = tags.downcast_into::().map_err(|_| Halt::Defer)?; + defaults.set_item(TAGS_KEY, PyList::new(py, tags.iter())?)?; + } + Ok(defaults) + } + + /// Accept hook keys `validate_model_hook_config` accepts. + fn check_hooks(&self, py: Python<'_>, values: &Bound<'_, PyDict>) -> Built<()> { + let entry_types = self.hook_entry_types.bind(py); + for key in HOOK_KEYS { + let Some(value) = values.get_item(key)? else { + continue; + }; + let entries = sequence_items(&value).ok_or(Halt::Defer)?; + for entry in entries { + if !entry.is_instance(entry_types)? { + return Err(Halt::Defer); + } + } + } + Ok(()) + } +} + +/// The template expansions of one model's build and the reads they recorded. +struct Expansion<'a, 'py> { + builder: &'a NativeModelConfigBuilder, + py: Python<'py>, + model_name: &'a str, + reads: Vec<(&'static str, String)>, +} + +impl<'py> Expansion<'_, 'py> { + fn expand( + &mut self, + values: &Bound<'py, PyDict>, + variables: &Bound<'py, PyDict>, + context: Bound<'py, PyDict>, + preserve_unknown_context: bool, + ) -> Built> { + let result = self.expand_value( + values.as_any(), + variables, + context, + preserve_unknown_context, + )?; + result.downcast_into::().map_err(|_| Halt::Defer) + } + + fn expand_value( + &mut self, + value: &Bound<'py, PyAny>, + variables: &Bound<'py, PyDict>, + context: Bound<'py, PyDict>, + preserve_unknown_context: bool, + ) -> Built> { + let environment = self.builder.environment.bind(self.py).clone(); + let host = PythonHost::new( + self.py, + TemplateSources(variables.clone(), environment, context), + ); + let options = TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context, + }; + let result = expanded(&host, value, options); + self.reads.extend(host.into_reads()); + Ok(result?) + } + + fn run_context(&self) -> Built> { + let context = PyDict::new(self.py); + context.set_item(RUN_ID_CONTEXT, self.builder.run_id.bind(self.py))?; + context.set_item( + RUN_TARGET_CONTEXT, + self.optional(self.builder.target_name.as_ref()), + )?; + Ok(context) + } + + fn optional(&self, value: Option<&Py>) -> Bound<'py, PyAny> { + value.map_or_else( + || self.py.None().into_bound(self.py), + |item| item.bind(self.py).clone(), + ) + } + + /// Return `build_model_context_values` for these values. + fn model_context( + &self, + values: &Bound<'py, PyDict>, + include_target_values: bool, + ) -> Built> { + let py = self.py; + let none = py.None().into_bound(py); + let database = string_entry(values, DATABASE_KEY)?; + let schema = string_entry(values, SCHEMA_KEY)?; + let alias = match string_entry(values, ALIAS_KEY)? { + Some(alias) => alias, + None => PyString::new(py, self.model_name).into_any(), + }; + let context = self.run_context()?; + context.set_item(MODEL_NAME_CONTEXT, self.model_name)?; + context.set_item(MODEL_DATABASE_CONTEXT, database.as_ref().unwrap_or(&none))?; + context.set_item(MODEL_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; + context.set_item(MODEL_ALIAS_CONTEXT, &alias)?; + if !include_target_values { + return Ok(context); + } + let qualified = match (&database, &schema) { + (Some(database), Some(schema)) => Some(format!( + "{}.{}.{}", + text_of(database)?, + text_of(schema)?, + text_of(&alias)? + )), + (None, Some(schema)) => Some(format!("{}.{}", text_of(schema)?, text_of(&alias)?)), + _ => None, + }; + context.set_item( + DESTINATION_DATABASE_CONTEXT, + database.as_ref().unwrap_or(&none), + )?; + context.set_item(DESTINATION_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; + context.set_item(DESTINATION_TABLE_CONTEXT, &alias)?; + context.set_item(DESTINATION_QUALIFIED_CONTEXT, qualified)?; + Ok(context) + } + + /// Check preserved namespaces and apply `apply_environment_database_schema_overrides`. + fn namespaced_values( + &mut self, + values: &Bound<'py, PyDict>, + logical_database: &Option>, + logical_schema: &Option>, + ) -> Built> { + let overridden = values.copy()?; + let Some((database, schema)) = &self.builder.target_namespace else { + return Ok(overridden); + }; + let py = self.py; + let namespace = [ + ( + DATABASE_KEY, + database.bind(py).clone(), + logical_database.is_some(), + ), + ( + SCHEMA_KEY, + schema.bind(py).clone(), + logical_schema.is_some(), + ), + ]; + let preserved = + |value: &Bound<'py, PyAny>| PyNode(value.clone()).is_text(PRESERVE_TARGET_VALUE); + if namespace + .iter() + .any(|(_, value, logical)| preserved(value) && !logical) + { + return Err(Halt::Defer); + } + let context = self.model_context(values, false)?; + for (key, value, _) in namespace { + if value.is_none() || preserved(&value) { + continue; + } + let variables = self.builder.variables.bind(py).clone(); + let expanded_value = self.expand_value(&value, &variables, context.copy()?, false)?; + overridden.set_item(key, expanded_value)?; + } + Ok(overridden) + } +} + +/// Merge `overlay` over a copy of `base` as `_merged_with_tag_union` does. +fn merged_layer<'py>( + py: Python<'py>, + base: &Bound<'py, PyDict>, + overlay: &Bound<'py, PyDict>, +) -> Built> { + let result = base.copy()?; + result.update(overlay.as_mapping())?; + if let (Some(base_tags), Some(overlay_tags)) = + (set_entry(base, TAGS_KEY)?, set_entry(overlay, TAGS_KEY)?) + { + let tags = merged_strings(&base_tags, &overlay_tags)?; + result.set_item(TAGS_KEY, PyList::new(py, tags)?)?; + } + if let (Some(base_columns), Some(overlay_columns)) = ( + set_entry(base, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, + set_entry(overlay, ROW_DIFF_EXCLUDE_COLUMNS_KEY)?, + ) { + let columns = merged_strings(&base_columns, &overlay_columns)?; + result.set_item(ROW_DIFF_EXCLUDE_COLUMNS_KEY, PyTuple::new(py, columns)?)?; + } + if let (Some(base_tolerances), Some(overlay_tolerances)) = ( + set_entry(base, ROW_DIFF_TOLERANCES_KEY)?, + set_entry(overlay, ROW_DIFF_TOLERANCES_KEY)?, + ) { + result.set_item( + ROW_DIFF_TOLERANCES_KEY, + merged_tolerances(&base_tolerances, &overlay_tolerances)?, + )?; + } + Ok(result) +} + +/// Return the strings of `base` then the new strings of `overlay`, as `_merge_string_sequence`. +fn merged_strings(base: &Bound<'_, PyAny>, overlay: &Bound<'_, PyAny>) -> Built> { + let mut merged = string_items(base)?; + for item in string_items(overlay)? { + if !merged.contains(&item) { + merged.push(item); + } + } + Ok(merged) +} + +/// Return `_as_string_list(value)` when every item is a string. +fn string_items(value: &Bound<'_, PyAny>) -> Built> { + let Some(items) = sequence_items(value) else { + return Ok(Vec::new()); + }; + items + .iter() + .map(|item| PyNode(item.clone()).text().ok_or(Halt::Defer)) + .collect() +} + +/// Deep-merge row-diff tolerances by section as `_merge_row_diff_tolerances_mapping` does. +fn merged_tolerances<'py>( + base: &Bound<'py, PyAny>, + overlay: &Bound<'py, PyAny>, +) -> Built> { + let (Ok(base), Ok(overlay)) = (base.downcast::(), overlay.downcast::()) else { + return Ok(overlay.clone()); + }; + let merged = base.copy()?; + for section in AUDIT_OVERRIDE_SECTIONS { + let Some(overlay_section) = set_entry(overlay, section)? else { + continue; + }; + let base_section = set_entry(base, section)?; + if let Some(base_section) = &base_section + && let (Ok(base_section), Ok(overlay_mapping)) = ( + base_section.downcast::(), + overlay_section.downcast::(), + ) + { + let section_values = base_section.copy()?; + section_values.update(overlay_mapping.as_mapping())?; + merged.set_item(section, section_values)?; + } else { + merged.set_item(section, &overlay_section)?; + } + } + for (key, value) in overlay.iter() { + if key.is_instance_of::() + && !AUDIT_OVERRIDE_SECTIONS + .iter() + .any(|section| PyNode(key.clone()).is_text(section)) + { + merged.set_item(key, value)?; + } + } + Ok(merged.into_any()) +} + +/// Return `values.get(key)` unless it is absent or `None`. +fn set_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built>> { + Ok(values.get_item(key)?.filter(|value| !value.is_none())) +} + +/// Return the value when it is a string, as `isinstance(raw, str)` keeps it. +fn string_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built>> { + Ok(values + .get_item(key)? + .filter(|value| value.is_instance_of::())) +} + +fn text_of(value: &Bound<'_, PyAny>) -> Built { + PyNode(value.clone()).text().ok_or(Halt::Defer) +} + +fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option>> { + if let Ok(items) = value.downcast::() { + Some(items.iter().collect()) + } else if let Ok(items) = value.downcast::() { + Some(items.iter().collect()) + } else { + None + } +} + +fn present(presence: Presence) -> Built { + match presence { + Presence::Present => Ok(true), + Presence::Absent => Ok(false), + Presence::Deferred => Err(Halt::Defer), + } +} + +fn entry_has_template(values: &Bound<'_, PyDict>, key: &str) -> Built { + match values.get_item(key)? { + Some(value) => present(contains_template(&PyNode(value))), + None => Ok(false), + } +} + +/// Accept header tags `_validate_model_header_tags` accepts. +fn check_header_tags(header: &Bound<'_, PyDict>) -> Built<()> { + let Some(tags) = set_entry(header, TAGS_KEY)? else { + return Ok(()); + }; + let tags = tags.downcast::().map_err(|_| Halt::Defer)?; + if tags.iter().all(|tag| tag.is_instance_of::()) { + Ok(()) + } else { + Err(Halt::Defer) + } +} + +/// Drop storage keys, then `full_refresh` where `build_model_config` drops it. +fn storage_free_values<'py>( + py: Python<'py>, + resolved: &Bound<'py, PyDict>, + header: &Bound<'py, PyDict>, +) -> Built> { + if let Some(materialized) = set_entry(resolved, MATERIALIZED_KEY)? + && !materialized.is_instance_of::() + { + return Err(Halt::Defer); + } + let values = PyDict::new(py); + for (key, value) in resolved.iter() { + let storage = [RETENTION_KEY, TABLE_TYPE_KEY] + .iter() + .any(|name| PyNode(key.clone()).is_text(name)); + if !storage { + values.set_item(key, value)?; + } + } + let incremental = values + .get_item(MATERIALIZED_KEY)? + .is_some_and(|value| PyNode(value).is_text(INCREMENTAL_MATERIALIZATION)); + if !header.contains(FULL_REFRESH_KEY)? && !incremental && values.contains(FULL_REFRESH_KEY)? { + values.del_item(FULL_REFRESH_KEY)?; + } + Ok(values) +} + +/// Defer when config outside hooks holds a macro call `validate_model_config_has_no_macros` rejects. +fn check_no_config_macros(py: Python<'_>, values: &Bound<'_, PyDict>) -> Built<()> { + let scanned = PyList::empty(py); + for (key, value) in values.iter() { + if !HOOK_KEYS + .iter() + .any(|hook| PyNode(key.clone()).is_text(hook)) + { + scanned.append(value)?; + } + } + if present(contains_macro_call(&PyNode(scanned.into_any())))? { + Err(Halt::Defer) + } else { + Ok(()) + } +} + +fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { + let mut keys = header + .keys() + .iter() + .map(|key| { + if key.is_exact_instance_of::() { + key.extract::().map_err(Halt::from) + } else { + Err(Halt::Defer) + } + }) + .collect::>>()?; + keys.sort(); + Ok(keys) +} + +fn built_tuple<'py>(py: Python<'py>, built: BuiltConfig<'py>) -> PyResult> { + let retention = match built.retention { + RetentionOverride::Inherit => None, + RetentionOverride::Unmanaged => Some((None, true)), + RetentionOverride::Days(days) => Some((Some(days), false)), + }; + let table_type = match built.table_type { + TableTypeOverride::Inherit => None, + TableTypeOverride::Permanent => Some("permanent"), + TableTypeOverride::Transient => Some("transient"), + }; + ( + built.values, + PyTuple::new(py, built.header_keys)?, + ( + built.logical_schema, + built.layer_schema_configured, + built.logical_database, + ), + (retention, table_type), + built.reads, + ) + .into_pyobject(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs new file mode 100644 index 0000000000..72aca263a1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -0,0 +1,98 @@ +//! The Python model validators' happy path, decided natively. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_model_config::model_validation::main::accept_model_config::accept_model_config; +use sqlbuild_model_config::model_validation::models::{ + ModelReference, ModelValidationFacts, ProjectValidationFacts, Rejected, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; + +/// Model, seed, source, function and table function names. +type ResourceNames = ( + HashSet, + HashSet, + HashSet, + HashSet, + HashSet, +); + +/// References, declared schema columns, unmanaged retention and declared table type. +type ModelFacts<'py> = (Bound<'py, PyTuple>, Option>, bool, bool); + +/// The project facts every model's validation reads, captured once per compile. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeModelValidator { + project: ProjectValidationFacts, +} + +#[pymethods] +impl NativeModelValidator { + #[new] + fn new( + names: ResourceNames, + custom_materializations: HashSet, + microbatch_concurrency: bool, + ) -> Self { + let (models, seeds, sources, functions, table_functions) = names; + Self { + project: ProjectValidationFacts { + custom_materializations, + microbatch_concurrency, + models, + seeds, + sources, + functions, + table_functions, + }, + } + } + + /// Return whether every Python model validator accepts this model; `False` runs them. + fn accepts( + &self, + values: Bound<'_, PyDict>, + model: (String, String), + facts: ModelFacts<'_>, + ) -> PyResult { + compiler_guard(|| { + let (model_name, query_sql) = model; + let (references, declared_columns, retention_unmanaged, table_type_declared) = facts; + let references = references + .iter() + .map(|reference| { + Ok(ModelReference { + kind: reference.getattr("ref_kind")?.extract()?, + name: reference.getattr("ref_name")?.extract()?, + }) + }) + .collect::>>()?; + let entries: Vec<(PyNode<'_>, PyNode<'_>)> = values + .iter() + .map(|(key, value)| (PyNode(key), PyNode(value))) + .collect(); + let facts = ModelValidationFacts { + model_name: &model_name, + references: &references, + declared_columns: declared_columns.as_deref(), + query_sql: &query_sql, + retention_unmanaged, + table_type_declared, + }; + Ok(match accept_model_config(entries, &self.project, &facts) { + Ok(()) => true, + Err(Rejected) => false, + }) + }) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 5c67efdfdd..0e31b5dade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -2,11 +2,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::model_config::{config_presence, config_templates, header_metadata}; +use crate::bindings::_helpers::model_config::{ + config_presence, config_templates, header_metadata, model_config_builder, model_validator, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { header_metadata::register(module)?; config_presence::register(module)?; config_templates::register(module)?; + model_config_builder::register(module)?; + model_validator::register(module)?; Ok(()) } diff --git a/scripts/compiler_differential/_helpers/corpus/config_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/config_failure_cases.py new file mode 100644 index 0000000000..688ad48e97 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/config_failure_cases.py @@ -0,0 +1,171 @@ +"""Minimal projects whose layered model config fails a model config validator.""" + +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, + staging_files, +) +from scripts.compiler_differential.constants import FAILURE_BASE_STAGING +from scripts.compiler_differential.models import FailureCase + +_STAGING_HEADER: str = '"Staged orders",' +_INCREMENTAL: str = ( + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor order_id,\n cursor_type integer,\n" +) +_WATERMARK: str = ( + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor placed_at,\n cursor_type timestamp,\n cursor_grain day,\n" + " incremental_mode microbatch,\n microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n batch_size 1d,\n" + " cursor_inputs (raw_orders (column placed_at, roles [filter, watermark])),\n" +) + + +def _staging_header(extra: str) -> dict[str, str]: + return staging_files( + FAILURE_BASE_STAGING.replace(_STAGING_HEADER, f"{_STAGING_HEADER}\n{extra}") + ) + + +def config_failure_cases() -> tuple[FailureCase, ...]: + """Return the model config validator failure cases in a stable order.""" + + return (*_layering_cases(), *_incremental_cases(), *_materialization_cases()) + + +def _layering_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="config-header-tags-not-list", + expected_code="P001", + expected_message="tags must be a list", + files=_staging_header(' tags "orders",'), + ), + failure_case( + name="config-untyped-hook-entry", + expected_code="P001", + expected_message="entries must use typed", + files=config_files('\n[path_defaults.staging]\npre_hooks = ["SELECT 1"]\n'), + ), + failure_case( + name="config-macro-outside-hooks", + expected_code="P001", + expected_message="does not allow macros", + files=_staging_header(' alias "@cents(amount)",'), + ), + failure_case( + name="config-unknown-table-type", + expected_code="P001", + expected_message="table_type must be permanent", + files=_staging_header(" materialized table,\n table_type temporary,"), + ), + failure_case( + name="config-retention-not-whole-days", + expected_code="P001", + expected_message="whole-day string", + files=_staging_header(" materialized table,\n time_travel_retention 6h,"), + ), + failure_case( + name="config-equally-specific-path-defaults", + expected_code="D007", + expected_message="equally specific path_defaults keys", + files=config_files( + '\n[path_defaults."*/stg_orders.sql"]\nmaterialized = "table"\n' + '\n[path_defaults."staging/*"]\nmaterialized = "view"\n' + ), + ), + ) + + +def _incremental_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="config-incremental-without-strategy", + expected_code="P001", + expected_message="requires incremental_strategy", + files=_staging_header(" materialized incremental,"), + ), + failure_case( + name="config-cursor-bounds-reversed", + expected_code="P001", + expected_message="cursor_start must be before exclusive cursor_end", + files=_staging_header(f"{_INCREMENTAL} cursor_start 20,\n cursor_end 10,"), + ), + failure_case( + name="config-microbatch-without-strategy", + expected_code="P001", + expected_message="requires explicit microbatch_strategy", + files=_staging_header(f"{_INCREMENTAL} incremental_mode microbatch,"), + ), + failure_case( + name="config-watermark-limit-below-lookback", + expected_code="P001", + expected_message="below the ordinary lookback requirement", + files=_staging_header(f"{_WATERMARK} lookback 3d,\n max_microbatches 2,"), + ), + failure_case( + name="config-concurrency-without-capability", + expected_code="P001", + expected_message="batch_concurrency > 1 requires", + files=_staging_header(f"{_INCREMENTAL} batch_concurrency 2,"), + ), + failure_case( + name="config-merge-exclusion-overlaps-key", + expected_code="P001", + expected_message="cannot include unique_key", + files=_staging_header( + " materialized incremental,\n incremental_strategy merge,\n" + " unique_key [order_id],\n merge_exclude_columns [ORDER_ID]," + ), + ), + ) + + +def _materialization_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="config-unknown-materialization", + expected_code="P001", + expected_message="unknown materialization 'archive'", + files=_staging_header(" materialized archive,"), + ), + failure_case( + name="config-unknown-contract", + expected_code="P001", + expected_message="unknown contract 'strict'", + files=_staging_header(" contract strict,"), + ), + failure_case( + name="config-incremental-key-on-table", + expected_code="P001", + expected_message="only valid for incremental models", + files=_staging_header(" materialized table,\n on_schema_change fail,"), + ), + failure_case( + name="config-snapshot-without-unique-key", + expected_code="P001", + expected_message="snapshot materialization requires unique_key", + files=_staging_header(" materialized snapshot,\n snapshot_strategy check,"), + ), + failure_case( + name="config-retention-on-view", + expected_code="P001", + expected_message="not valid for views", + files=_staging_header(" materialized view,\n time_travel_retention 7d,"), + ), + failure_case( + name="config-migrate-force-on-table", + expected_code="P001", + expected_message="migrate_force is only valid", + files=_staging_header( + " materialized table,\n migrate_from legacy_orders,\n migrate_force true," + ), + ), + failure_case( + name="config-old-name-view-without-duration", + expected_code="P001", + expected_message="old_name_view must be a positive duration", + files=_staging_header(" materialized table,\n old_name_view soon,"), + ), + ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index f041e96d90..d0fb4d2c30 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -8,6 +8,9 @@ mart_body_files, staging_files, ) +from scripts.compiler_differential._helpers.corpus.config_failure_cases import ( + config_failure_cases, +) from scripts.compiler_differential._helpers.corpus.discovery_failure_cases import ( discovery_failure_cases, ) @@ -46,6 +49,7 @@ def all_failure_cases() -> tuple[FailureCase, ...]: *_compile_failure_cases(), *discovery_failure_cases(), *render_failure_cases(), + *config_failure_cases(), *scope_failure_cases(), *reference_failure_cases(), *analysis_failure_cases(), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 48897e6103..5557fc15a2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -166,6 +166,42 @@ def expand_config_templates( flags: tuple[bool, bool, bool], ) -> tuple[object, list[tuple[str, str]]] | str: ... +class NativeModelConfigBuilder: + def __init__( + self, + layers: tuple[dict[str, object], dict[str, dict[str, object]], tuple[type, ...]], + sources: tuple[dict[str, object], object], + run: tuple[str | None, str], + target_namespace: tuple[str | None, str | None] | None, + ) -> None: ... + def path_default(self, model_path: str) -> tuple[bool, str | None]: ... + def build( + self, header: dict[str, object], matched_path_default: str | None, model_name: str + ) -> ( + tuple[ + dict[str, object], + tuple[str, ...], + tuple[str | None, bool, str | None], + tuple[tuple[int | None, bool] | None, str | None], + list[tuple[str, str]], + ] + | None + ): ... + +class NativeModelValidator: + def __init__( + self, + names: tuple[set[str], set[str], set[str], set[str], set[str]], + custom_materializations: set[str], + microbatch_concurrency: bool, + ) -> None: ... + def accepts( + self, + values: dict[str, object], + model: tuple[str, str], + facts: tuple[tuple[object, ...], list[str] | None, bool, bool], + ) -> bool: ... + class SqlReferenceScanner: def __init__(self, syntax: dict[str, object]) -> None: ... def extract( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 08929955a1..17f5a3e6dd 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -19,12 +19,16 @@ _validate_model_header_tags, build_layered_model_values, build_model_header_schema_entry, + build_native_model_config, contains_config_macro_calls, contains_config_templates, find_matching_path_default, find_schema_model_match, + native_model_config_session, + native_path_default, strip_model_header_metadata_from_config, validate_declared_schema_models_are_attached, + validate_model_config, validate_no_macros_in_config_value, ) from sqlbuild.compiler.compile._helpers.attachment.references import ( @@ -35,17 +39,6 @@ build_known_table_function_names, validate_model_references, ) -from sqlbuild.compiler.compile._helpers.config.model_validation import ( - validate_contract_config, - validate_custom_materialization_config, - validate_incremental_config, - validate_microbatch_project_capability, - validate_model_migration_config, - validate_non_incremental_config, - validate_placeholder_config, - validate_snapshot_config, - validate_storage_policies, -) from sqlbuild.compiler.compile._helpers.config.namespace_validation import ( validate_preserved_logical_namespace, ) @@ -124,6 +117,11 @@ ModelConfigScanCache, ModelHeaderColumnCache, ModelInputBuildContext, + ModelResourceNames, + ModelValidationRequest, + ModelValidatorContext, + NativeModelConfigInputs, + NativeModelConfigSession, SqlAnalysisOptOutRequest, SqlReferenceOrigin, SqlReferenceScan, @@ -270,6 +268,7 @@ class _ModelValidationContext: known_function_names: set[str] known_table_function_names: set[str] custom_materialization_names: frozenset[str] + validators: ModelValidatorContext @dataclass(frozen=True) @@ -492,6 +491,15 @@ def _build_model_inputs( custom_materialization_names: frozenset[str] = frozenset( mf.name for mf in discovered_inputs.materialization_files ) + native_model_config: bool = native_stage_enabled(NativeStage.MODEL_CONFIG) + names: ModelResourceNames = ModelResourceNames( + models=known_model_names, + seeds=known_seed_names, + sources=known_source_names, + functions=known_function_names, + table_functions=known_table_function_names, + custom_materializations=custom_materialization_names, + ) validation_context: _ModelValidationContext = _ModelValidationContext( effective_settings=effective_settings, no_sql_validation=no_sql_validation, @@ -505,6 +513,26 @@ def _build_model_inputs( known_function_names=known_function_names, known_table_function_names=known_table_function_names, custom_materialization_names=custom_materialization_names, + validators=ModelValidatorContext( + names=names, + settings=effective_settings, + external_sql_reference_resolver=external_sql_reference_resolver, + native_config=( + native_model_config_session( + inputs=NativeModelConfigInputs( + project_config=discovered_inputs.project_config, + target_config=context.target_config, + effective_vars=effective_vars, + effective_target_name=context.effective_target_name, + run_id=context.run_id, + microbatch_concurrency=effective_settings.microbatch_concurrency, + ), + names=names, + ) + if native_model_config + else None + ), + ), ) sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( discovered_inputs.sql_hook_files @@ -524,7 +552,6 @@ def _build_model_inputs( strict=True, ) ) - native_model_config: bool = native_stage_enabled(NativeStage.MODEL_CONFIG) prepared_files: tuple[DiscoveredSqlModelFile, ...] = ( tuple( model_file @@ -619,14 +646,31 @@ def _build_model_input( declarations: _VisibleModelDeclarations = declaration_cache.for_model( model_file=model_file, consumer=model_identity ) - matched_path_default: str | None = find_matching_path_default( - model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, + native_config: NativeModelConfigSession | None = validation_context.validators.native_config + matched_path_default: str | None = ( + find_matching_path_default( + model_file=model_file, + path_defaults=discovered_inputs.project_config.path_defaults, + ) + if native_config is None + else native_path_default(session=native_config, model_file=model_file) ) effective_config: CompileModelConfig | None = reusable_config_cache.get( matched_path_default=matched_path_default, model_header_values=model_file.header_values, ) + if effective_config is None and native_config is not None: + effective_config = build_native_model_config( + session=native_config, + model_file=model_file, + matched_path_default=matched_path_default, + ) + if effective_config is not None: + reusable_config_cache.remember( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + config=effective_config, + ) if effective_config is None: effective_config = build_model_config( request=ModelConfigBuildRequest( @@ -986,47 +1030,15 @@ def _validate_model_input( references=reference_scan.references, argument_references=argument_references, ) - validate_model_references( - references=references, - model_file=model_file, - known_model_names=context.known_model_names, - known_seed_names=context.known_seed_names, - known_source_names=context.known_source_names, - known_function_names=context.known_function_names, - known_table_function_names=context.known_table_function_names, - external_sql_reference_resolver=context.external_sql_reference_resolver, - ) - validate_incremental_config( - config=config, - model_name=model_name, - ref_count=len(references), - known_input_names=frozenset(reference.ref_name for reference in references), - declared_columns=model_schema_columns, - ) - validate_microbatch_project_capability( - config=config, - settings=context.effective_settings, - model_name=model_name, - ) - validate_contract_config(config=config, model_name=model_name) - validate_non_incremental_config(config=config, model_name=model_name) - validate_snapshot_config( - config=config, - model_name=model_name, - declared_columns=model_schema_columns, - ) - validate_custom_materialization_config( - config=config, - model_name=model_name, - custom_materialization_names=context.custom_materialization_names, - ) - validate_storage_policies(config=config, model_name=model_name) - validate_model_migration_config(config=config, model_name=model_name) - validate_placeholder_config( - config=config, - model_name=model_name, - query_sql=expanded_query_sql, - custom_materialization_names=context.custom_materialization_names, + validate_model_config( + context=context.validators, + request=ModelValidationRequest( + model_file=model_file, + config=config, + references=references, + declared_columns=model_schema_columns, + query_sql=expanded_query_sql, + ), ) return ( sql_validation_enabled and not reference_scan.invalid_calls, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index dd21579021..5ba375aded 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -2,12 +2,15 @@ from __future__ import annotations +import os from dataclasses import replace from pathlib import Path from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.authored_values.main._optional_named_string import optional_named_string +from sqlbuild.compiler.compile._helpers.attachment.references import validate_model_references from sqlbuild.compiler.compile._helpers.audit_factories.core import ( merge_validated_model_audits, parse_model_header_audit_factories, @@ -17,7 +20,19 @@ ) from sqlbuild.compiler.compile._helpers.config.model_validation import ( validate_column_migration_config, + validate_contract_config, + validate_custom_materialization_config, + validate_incremental_config, + validate_microbatch_project_capability, + validate_model_migration_config, + validate_non_incremental_config, + validate_placeholder_config, + validate_snapshot_config, + validate_storage_policies, ) +from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention +from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type +from sqlbuild.compiler.compile._helpers.render.context_templates import record_template_reads from sqlbuild.compiler.compile._helpers.render.templating import ( contains_template_data, ) @@ -34,6 +49,11 @@ IdentityPresenceCache, ModelConfigScanCache, ModelHeaderColumnCache, + ModelResourceNames, + ModelValidationRequest, + ModelValidatorContext, + NativeModelConfigInputs, + NativeModelConfigSession, ) from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns from sqlbuild.compiler.discovery.models import ( @@ -41,6 +61,9 @@ DiscoveredSchemaFile, DiscoveredSqlModelFile, ModelSchemaDeclaration, + NamedSqlHookEntry, + PythonHookEntry, + SqlHookEntry, ) from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError from sqlbuild.compiler.model_config.main._native_config_contains_macro_call import ( @@ -51,17 +74,39 @@ ) from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.compiler.path_defaults.main._select import select_path_default +from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.spec.contracts.models import ( DefaultsConfig, + MaterializationDefaultsConfig, + ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, SchemaAuditInstance, SchemaColumn, SchemaDynamicColumnFamily, SchemaModelEntry, SettingsConfig, SourceLocation, + TargetConfig, ) +from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) +_TABLE_BACKED_MATERIALIZATIONS: tuple[str, ...] = ( + MaterializationType.TABLE, + MaterializationType.INCREMENTAL, + MaterializationType.SNAPSHOT, +) +_MATERIALIZED_CONFIG_KEY: str = "materialized" +type _BuiltNamespace = tuple[str | None, bool, str | None] +type _StorageOverrides = tuple[tuple[int | None, bool] | None, str | None] +type _BuiltConfig = tuple[ + dict[str, object], + tuple[str, ...], + _BuiltNamespace, + _StorageOverrides, + list[tuple[str, str]], +] _MODEL_DESCRIPTION_KEY: str = "description" _MODEL_SCHEMA_KEY: str = "model_schema" @@ -811,3 +856,195 @@ def _model_sql_validation_gate( model_config=model_config, ) ) + + +def native_model_config_session( + *, inputs: NativeModelConfigInputs, names: ModelResourceNames +) -> NativeModelConfigSession: + """Capture the project inputs every model's native config build and validation read.""" + + project_config: ProjectConfig = inputs.project_config + target: TargetConfig | None = inputs.target_config + materialization_defaults: MaterializationDefaultsConfig = ( + project_config.materialization_defaults or MaterializationDefaultsConfig() + ) + return NativeModelConfigSession( + path_defaults=project_config.path_defaults, + builder=_native.NativeModelConfigBuilder( + ( + project_defaults_to_mapping(project_config.defaults), + project_config.path_defaults, + (SqlHookEntry, NamedSqlHookEntry, PythonHookEntry), + ), + (inputs.effective_vars, os.environ), + (inputs.effective_target_name, inputs.run_id), + None if target is None else (target.database, target.schema), + ), + validator=_native.NativeModelValidator( + (names.models, names.seeds, names.sources, names.functions, names.table_functions), + set(names.custom_materializations), + inputs.microbatch_concurrency, + ), + inherited_storage={ + materialized: ( + resolve_time_travel_retention( + materialized=materialized, + model_value=None, + materialization_defaults=materialization_defaults, + target_config=target, + model_name="", + ), + resolve_table_type( + materialized=materialized, + model_value=None, + materialization_defaults=materialization_defaults, + target_config=target, + model_name="", + ), + ) + for materialized in (None, *_TABLE_BACKED_MATERIALIZATIONS) + }, + ) + + +def native_path_default( + *, session: NativeModelConfigSession, model_file: DiscoveredSqlModelFile +) -> str | None: + """Select a model's path default natively; Python reports equally specific matches.""" + + selected, key = session.builder.path_default(str(model_file.relative_path)) + if selected: + return key + return find_matching_path_default(model_file=model_file, path_defaults=session.path_defaults) + + +def build_native_model_config( + *, + session: NativeModelConfigSession, + model_file: DiscoveredSqlModelFile, + matched_path_default: str | None, +) -> CompileModelConfig | None: + """Build a model's effective config natively, or return None where Python must build it.""" + + built: _BuiltConfig | None = session.builder.build( + model_file.header_values, matched_path_default, model_file.file_path.stem + ) + if built is None: + return None + values, header_keys, namespace, overrides, reads = built + logical_schema, layer_schema_configured, logical_database = namespace + retention_override, table_type_override = overrides + record_template_reads(tuple(reads)) + materialized: object | None = values.get(_MATERIALIZED_CONFIG_KEY) + inherited_retention, inherited_table_type = session.inherited_storage[ + materialized + if isinstance(materialized, str) and materialized in _TABLE_BACKED_MATERIALIZATIONS + else None + ] + return CompileModelConfig( + values=values, + model_header_keys=header_keys, + matched_path_default=matched_path_default, + logical_schema=logical_schema, + layer_schema=logical_schema if layer_schema_configured else None, + logical_database=logical_database, + time_travel_retention=( + inherited_retention + if retention_override is None + else ResolvedTimeTravelRetention( + desired_days=retention_override[0], + unmanaged=retention_override[1], + source=TimeTravelRetentionSource.MODEL, + ) + ), + table_type=( + inherited_table_type + if table_type_override is None + else ResolvedTableType( + value=TableType(table_type_override), + source=TableTypeSource.MODEL, + declared=True, + ) + ), + ) + + +def native_model_validators_accept( + *, session: NativeModelConfigSession, request: ModelValidationRequest +) -> bool: + """Return whether every Python model validator accepts the model; False runs them.""" + + config: CompileModelConfig = request.config + return session.validator.accepts( + config.values, + (request.model_file.file_path.stem, request.query_sql), + ( + request.references, + ( + None + if request.declared_columns is None + else [column.name for column in request.declared_columns] + ), + config.time_travel_retention.unmanaged, + config.table_type.declared, + ), + ) + + +def validate_model_config( + *, context: ModelValidatorContext, request: ModelValidationRequest +) -> None: + """Validate one model; a native acceptance skips Python, any rejection runs it.""" + + if context.native_config is not None and native_model_validators_accept( + session=context.native_config, request=request + ): + return + run_python_model_validators(context=context, request=request) + + +def run_python_model_validators( + *, context: ModelValidatorContext, request: ModelValidationRequest +) -> None: + """Run every Python model validator in order, raising the first error.""" + + names: ModelResourceNames = context.names + model_name: str = request.model_file.file_path.stem + validate_model_references( + references=request.references, + model_file=request.model_file, + known_model_names=names.models, + known_seed_names=names.seeds, + known_source_names=names.sources, + known_function_names=names.functions, + known_table_function_names=names.table_functions, + external_sql_reference_resolver=context.external_sql_reference_resolver, + ) + validate_incremental_config( + config=request.config, + model_name=model_name, + ref_count=len(request.references), + known_input_names=frozenset(reference.ref_name for reference in request.references), + declared_columns=request.declared_columns, + ) + validate_microbatch_project_capability( + config=request.config, settings=context.settings, model_name=model_name + ) + validate_contract_config(config=request.config, model_name=model_name) + validate_non_incremental_config(config=request.config, model_name=model_name) + validate_snapshot_config( + config=request.config, model_name=model_name, declared_columns=request.declared_columns + ) + validate_custom_materialization_config( + config=request.config, + model_name=model_name, + custom_materialization_names=names.custom_materializations, + ) + validate_storage_policies(config=request.config, model_name=model_name) + validate_model_migration_config(config=request.config, model_name=model_name) + validate_placeholder_config( + config=request.config, + model_name=model_name, + query_sql=request.query_sql, + custom_materialization_names=names.custom_materializations, + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index b8eae7c29b..90b33177da 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -41,6 +41,7 @@ ) from sqlbuild.cursor_algebra.constants import GRAIN_BATCH_SIZE from sqlbuild.cursor_algebra.models import Duration +from sqlbuild.errors.setting_help.main.model_header_help import model_header_help from sqlbuild.spec.contracts.constants import ( CURSOR_POLICY_DISABLED, EFFECTIVE_BATCH_SIZE_TOKEN, @@ -867,8 +868,12 @@ def _validate_incremental_cursor_rules( if ( cursor_start is not None and cursor_end is not None - and _cursor_contract_key(value=cursor_start, cursor_type=cursor_type) - >= _cursor_contract_key(value=cursor_end, cursor_type=cursor_type) + and _cursor_contract_key( + value=cursor_start, cursor_type=cursor_type, key="cursor_start", model_name=model_name + ) + >= _cursor_contract_key( + value=cursor_end, cursor_type=cursor_type, key="cursor_end", model_name=model_name + ) ): raise CompileInputError( f"model '{model_name}': cursor_start must be before exclusive cursor_end" @@ -883,15 +888,26 @@ def _validate_incremental_cursor_rules( raise CompileInputError(f"model '{model_name}': append_cursor_inclusive requires cursor") -def _cursor_contract_key(*, value: object, cursor_type: str | None) -> Decimal: +def _cursor_contract_key( + *, value: object, cursor_type: str | None, key: str, model_name: str +) -> Decimal: if cursor_type == CursorType.INTEGER: return Decimal(str(value)) parsed: datetime = datetime.fromisoformat(str(value)) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=UTC) - else: - parsed = parsed.astimezone(UTC) - return Decimal(str(parsed.timestamp())) + return Decimal(str(parsed.replace(tzinfo=UTC).timestamp())) + try: + utc_parsed: datetime = parsed.astimezone(UTC) + except OverflowError: + raise CompileInputError( + f"model '{model_name}': {key} value '{value}' falls outside years 1-9999 once " + "converted to UTC", + help=model_header_help( + purpose=f"keep {key} within years 1-9999 in UTC", + entry=f"{key} '{parsed.replace(tzinfo=UTC).isoformat()}'", + ), + ) from None + return Decimal(str(utc_parsed.timestamp())) def _validate_cursor_safety_overrides( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index d5b6ff54c9..a01d37e37f 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -266,7 +266,7 @@ def expand_config_templates( else None ) if isinstance(outcome, NativeTemplateExpansion): - _record_template_reads(outcome.reads) + record_template_reads(outcome.reads) return outcome.value expanded: object = expand_template_data( value=value, @@ -285,7 +285,9 @@ def expand_config_templates( return expanded -def _record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: +def record_template_reads(reads: tuple[tuple[str, str], ...]) -> None: + """Record native template environment and context reads for project reuse, in order.""" + for kind, name in reads: if kind == ENVIRONMENT_READ: COMPILE_INPUT_READS.environment_read(name) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index d091fb5b1e..62a07a1fcc 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1640,6 +1640,61 @@ class ModelConfigScanCache: native: bool = False +@dataclass(frozen=True) +class NativeModelConfigSession: + """The native config builder, validator, and inherited storage policies for one compile.""" + + path_defaults: dict[str, dict[str, object]] + builder: _native.NativeModelConfigBuilder + validator: _native.NativeModelValidator + inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]] + + +@dataclass(frozen=True) +class NativeModelConfigInputs: + """The project and run inputs every model's native config build reads.""" + + project_config: ProjectConfig + target_config: TargetConfig | None + effective_vars: dict[str, object] + effective_target_name: str | None + run_id: str + microbatch_concurrency: bool + + +@dataclass(frozen=True) +class ModelResourceNames: + """Discovered resource names the native reference check reads.""" + + models: set[str] + seeds: set[str] + sources: set[str] + functions: set[str] + table_functions: set[str] + custom_materializations: frozenset[str] + + +@dataclass(frozen=True) +class ModelValidationRequest: + """One model's effective config and the facts its validators read.""" + + model_file: DiscoveredSqlModelFile + config: CompileModelConfig + references: tuple[CompileSqlReference, ...] + declared_columns: tuple[SchemaColumn, ...] | None + query_sql: str + + +@dataclass(frozen=True) +class ModelValidatorContext: + """The project facts the model validators read, and the native session in preview.""" + + names: ModelResourceNames + settings: SettingsConfig + external_sql_reference_resolver: ExternalSqlReferenceResolver | None + native_config: NativeModelConfigSession | None + + @dataclass(frozen=True) class ModelConfigBuildRequest: """Cohesive inputs for one effective model-configuration build.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index a62336785f..45958b6c3a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1315,3 +1315,15 @@ class UntypedCursorOverrideSelectionE2ETestCase: command: tuple[str, ...] expected_exit_code: int expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class ModelConfigEnginesBuildE2ETestCase: + """One compiler engine building a project whose models layer and validate config.""" + + description: str + engine: str + overrides: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_relations: tuple[tuple[str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py new file mode 100644 index 0000000000..d0f5288fa0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py @@ -0,0 +1,198 @@ +"""E2E: layered, templated and validated model config builds the same on every compiler engine. + +The preview engine builds model config (path defaults, templates, storage policies) and checks the +model validators natively; a validator rejection re-runs Python for the exact error. Every engine +must build the same relations and report the same validator errors. +""" + +from __future__ import annotations + +from pathlib import Path +from subprocess import CompletedProcess +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + ModelConfigEnginesBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +_MART_SCHEMA_ENV_VAR: str = "SQB_MART_SCHEMA" +_INCREMENTAL_MODEL: str = "models/marts/orders_incremental.sql" +_SNAPSHOT_MODEL: str = "models/marts/order_history.sql" +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = "orders.duckdb"\n\n' + '[targets.dev]\nconnection = "local"\n\n' + '[defaults]\nmaterialized = "view"\ntags = ["core"]\n\n' + '[path_defaults.marts]\nmaterialized = "table"\n' + 'schema = "${coalesce(ENV:SQB_MART_SCHEMA, \'marts\')}"\ntags = ["marts"]\n\n' + '[path_defaults."staging/*"]\nschema = "staging"\n' + ), + "models/staging/stg_orders.sql": ( + 'MODEL (description "Staged orders.");\n\n' + "SELECT * FROM (VALUES (1, 10, 'placed'), (2, 20, 'shipped'))" + " AS t(order_id, amount, status)\n" + ), + "models/marts/order_totals.sql": ( + 'MODEL (\n description "Order totals for ${CTX:run.target}.",\n' + ' alias "${CTX:model.name}_v2",\n tags ["daily"],\n);\n\n' + 'SELECT status, SUM(amount) AS total_amount\nFROM __ref("stg_orders")\nGROUP BY status\n' + ), + _INCREMENTAL_MODEL: ( + 'MODEL (\n description "Orders loaded incrementally.",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + ' cursor order_id,\n cursor_type integer,\n cursor_start "0",\n cursor_end 100,\n' + " unique_key [order_id],\n);\n\n" + 'SELECT order_id, amount\nFROM __ref("stg_orders")\n' + "WHERE order_id >= __cursor_start() AND order_id < __cursor_end()\n" + ), + _SNAPSHOT_MODEL: ( + 'MODEL (\n description "Order status history.",\n materialized snapshot,\n' + " unique_key [order_id],\n snapshot_strategy check,\n check_columns [status],\n);\n\n" + 'SELECT order_id, status\nFROM __ref("stg_orders")\n' + ), +} +_REVERSED_CURSOR_BOUNDS: dict[str, str] = { + _INCREMENTAL_MODEL: _PROJECT_FILES[_INCREMENTAL_MODEL].replace( + "cursor_end 100", "cursor_end -5" + ) +} +_SNAPSHOT_WITHOUT_KEY: dict[str, str] = { + _SNAPSHOT_MODEL: _PROJECT_FILES[_SNAPSHOT_MODEL].replace(" unique_key [order_id],\n", "") +} +_BUILT_RELATIONS: tuple[tuple[str, str], ...] = ( + ("analytics", "order_history"), + ("analytics", "order_totals_v2"), + ("analytics", "orders_incremental"), + ("staging", "stg_orders"), +) +_PASSING_FRAGMENTS: tuple[str, ...] = ("PASS=4 WARN=0 FAIL=0",) +_REVERSED_BOUNDS_FRAGMENTS: tuple[str, ...] = ( + "error[P001]: model 'orders_incremental': cursor_start must be before exclusive cursor_end", +) +_SNAPSHOT_WITHOUT_KEY_FRAGMENTS: tuple[str, ...] = ( + "error[P001]: model 'order_history': snapshot materialization requires unique_key", +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelConfigEnginesBuildE2ETestCase( + description="layered and templated config builds on python", + engine="python", + overrides={}, + expected_exit_code=0, + expected_output_fragments=_PASSING_FRAGMENTS, + expected_relations=_BUILT_RELATIONS, + ), + ModelConfigEnginesBuildE2ETestCase( + description="reversed cursor bounds are the Python error on python", + engine="python", + overrides=_REVERSED_CURSOR_BOUNDS, + expected_exit_code=1, + expected_output_fragments=_REVERSED_BOUNDS_FRAGMENTS, + expected_relations=(), + ), + ModelConfigEnginesBuildE2ETestCase( + description="a snapshot without a unique key is the Python error on python", + engine="python", + overrides=_SNAPSHOT_WITHOUT_KEY, + expected_exit_code=1, + expected_output_fragments=_SNAPSHOT_WITHOUT_KEY_FRAGMENTS, + expected_relations=(), + ), + ModelConfigEnginesBuildE2ETestCase( + description="layered and templated config builds on native", + engine="native", + overrides={}, + expected_exit_code=0, + expected_output_fragments=_PASSING_FRAGMENTS, + expected_relations=_BUILT_RELATIONS, + ), + ModelConfigEnginesBuildE2ETestCase( + description="reversed cursor bounds are the Python error on native", + engine="native", + overrides=_REVERSED_CURSOR_BOUNDS, + expected_exit_code=1, + expected_output_fragments=_REVERSED_BOUNDS_FRAGMENTS, + expected_relations=(), + ), + ModelConfigEnginesBuildE2ETestCase( + description="a snapshot without a unique key is the Python error on native", + engine="native", + overrides=_SNAPSHOT_WITHOUT_KEY, + expected_exit_code=1, + expected_output_fragments=_SNAPSHOT_WITHOUT_KEY_FRAGMENTS, + expected_relations=(), + ), + ModelConfigEnginesBuildE2ETestCase( + description="layered and templated config builds on native-preview", + engine="native-preview", + overrides={}, + expected_exit_code=0, + expected_output_fragments=_PASSING_FRAGMENTS, + expected_relations=_BUILT_RELATIONS, + ), + ModelConfigEnginesBuildE2ETestCase( + description="reversed cursor bounds are the Python error on native-preview", + engine="native-preview", + overrides=_REVERSED_CURSOR_BOUNDS, + expected_exit_code=1, + expected_output_fragments=_REVERSED_BOUNDS_FRAGMENTS, + expected_relations=(), + ), + ModelConfigEnginesBuildE2ETestCase( + description="a snapshot without a unique key is the Python error on native-preview", + engine="native-preview", + overrides=_SNAPSHOT_WITHOUT_KEY, + expected_exit_code=1, + expected_output_fragments=_SNAPSHOT_WITHOUT_KEY_FRAGMENTS, + expected_relations=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_layered_model_config_when_building_on_each_engine_then_results_match( + test_case: ModelConfigEnginesBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={**_PROJECT_FILES, **test_case.overrides}, + ) + + result: CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine, _MART_SCHEMA_ENV_VAR: "analytics"}, + ) + output: str = result.stdout + result.stderr + execute_duckdb(db_path=project_dir / "orders.duckdb", sql="SELECT 1") + relations: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", + sql=( + "SELECT table_schema, table_name FROM information_schema.tables " + "WHERE table_schema IN ('analytics', 'staging') " + "AND NOT starts_with(table_name, '_sqlbuild') ORDER BY 1, 2" + ), + ) + + assert result.returncode == test_case.expected_exit_code, output + assert [fragment in output for fragment in test_case.expected_output_fragments] == [True] * len( + test_case.expected_output_fragments + ), output + assert tuple(relations) == test_case.expected_relations + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/__init__.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py new file mode 100644 index 0000000000..aaaaac98ca --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py @@ -0,0 +1,48 @@ +"""Test case types for the native model config build and validator parity tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ModelValidationParityTestCase: + """Seeded effective configs validated natively and by the Python validators.""" + + description: str + seed: int + count: int + expected_minimum_native_accepted: int + expected_minimum_python_rejected: int + expected_minimum_acceptance_percent: int + + +@dataclass(frozen=True) +class ModelConfigBuildParityTestCase: + """Seeded defaults, path defaults, headers and targets built natively and in Python.""" + + description: str + seed: int + count: int + expected_minimum_built: int + expected_minimum_python_raised: int + expected_minimum_build_percent: int + + +@dataclass(frozen=True) +class ModelConfigProjectTestCase: + """A whole project compiled and built on DuckDB under one compiler engine.""" + + description: str + engine: str + expected_models: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeRejectionParityTestCase: + """A config native validation must reject so Python reports its exact outcome.""" + + description: str + values: dict[str, object] + expected_native_accepted: bool + expected_python_error: str diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py new file mode 100644 index 0000000000..f33aa3c4b9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -0,0 +1,627 @@ +"""Seeded model configs, layers and targets compared between the native and Python paths.""" + +from __future__ import annotations + +import random +from dataclasses import dataclass, replace +from datetime import date +from itertools import compress +from operator import itemgetter +from pathlib import Path +from typing import cast + +from sqlbuild.compiler.compile._helpers.attachment.core import build_model_config +from sqlbuild.compiler.compile._helpers.attachment.model_config import ( + build_native_model_config, + find_matching_path_default, + native_model_config_session, + native_model_validators_accept, + native_path_default, + run_python_model_validators, +) +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompileModelConfig, + CompileSqlReference, + ModelConfigBuildRequest, + ModelResourceNames, + ModelValidationRequest, + ModelValidatorContext, + NativeModelConfigInputs, + NativeModelConfigSession, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredSqlModelFile, + NamedSqlHookEntry, + PythonHookEntry, + SqlHookEntry, +) +from sqlbuild.spec.contracts.models import ( + AuthoredTimeTravelRetention, + DefaultsConfig, + MaterializationDefaultsConfig, + MaterializationRetentionDefaults, + ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, + SchemaColumn, + SettingsConfig, + TargetConfig, +) +from sqlbuild.spec.contracts.types import TableType, TableTypeSource + +_ABSENT: object = object() +_RAISES: str = "Python raises" +_DEFERRED: str = "native defers" +_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) +_RUN_ID: str = "20261008T000000Z_orders" +BUILD_ENVIRONMENT: dict[str, str] = {"SQB_BUILD_DB": "ops", "SQB_BUILD_SCHEMA": "nightly"} +_NAMES: ModelResourceNames = ModelResourceNames( + models={"orders", "customers"}, + seeds={"regions"}, + sources={"raw.orders"}, + functions={"cents", "order_lines"}, + table_functions={"order_lines"}, + custom_materializations=frozenset({"ledger"}), +) +_VALIDATOR_CONTEXT: ModelValidatorContext = ModelValidatorContext( + names=_NAMES, + settings=SettingsConfig(), + external_sql_reference_resolver=None, + native_config=None, +) + + +def _reference(kind: str, name: str) -> CompileSqlReference: + return CompileSqlReference(ref_kind=kind, ref_name=name) + + +_ORDERS: CompileSqlReference = _reference("ref", "orders") +_REFERENCES: tuple[CompileSqlReference, ...] = ( + _ORDERS, + _reference("ref", "customers"), + _reference("ref", "invoices"), + _reference("seed", "regions"), + _reference("seed", "orders"), + _reference("source", "raw.orders"), + _reference("source", "raw.refunds"), + _reference("udf", "cents"), + _reference("udf", "order_lines"), + _reference("table_fn", "order_lines"), + _reference("table_fn", "cents"), + _reference("dbt_ref", "orders"), +) +_WATERMARK_INPUTS: dict[str, object] = { + "orders": {"column": "updated_at", "roles": ["filter", "watermark"]} +} +_INCREMENTAL: dict[str, object] = { + "materialized": "incremental", + "incremental_strategy": "append", + "cursor": "updated_at", + "cursor_type": "timestamp", + "cursor_grain": "day", +} +_PROFILES: tuple[tuple[dict[str, object], tuple[CompileSqlReference, ...]], ...] = ( + ({"materialized": "table", "tags": ["core"]}, (_ORDERS,)), + ({"materialized": "view", "contract": "none"}, ()), + ({**_INCREMENTAL, "cursor_start": "2024-01-01", "cursor_end": "2024-03-01"}, (_ORDERS,)), + ( + { + **_INCREMENTAL, + "incremental_strategy": "delete_insert", + "incremental_mode": "microbatch", + "microbatch_strategy": "watermark", + "cursor_watermark_mode": "all", + "batch_size": "1d", + "lookback": "2d", + "cursor_inputs": _WATERMARK_INPUTS, + "microbatch_limit": {"max_batches": 6, "action": "cap_from_start"}, + }, + (_ORDERS,), + ), + ( + { + **_INCREMENTAL, + "incremental_mode": "microbatch", + "microbatch_strategy": "rolling_window", + "cursor_inputs": {"orders": "updated_at"}, + }, + (_ORDERS,), + ), + ( + { + "materialized": "incremental", + "incremental_strategy": "merge", + "unique_key": ["order_id"], + "merge_exclude_columns": ["loaded_at"], + "on_schema_change": "append_new_columns", + "replay_on_change": "bounded-14d", + }, + (_ORDERS,), + ), + ( + { + "materialized": "incremental", + "incremental_strategy": "delete_insert", + "cursor": "order_id", + "cursor_type": "integer", + "cursor_start": "10", + "cursor_end": 20, + "unique_key": "order_id", + }, + (_ORDERS,), + ), + ( + { + "materialized": "snapshot", + "unique_key": "order_id", + "snapshot_strategy": "timestamp", + "updated_at": "updated_at", + "observed_at": "loaded_at", + "historical_input": "snapshot", + }, + (_ORDERS,), + ), + ( + { + "materialized": "snapshot", + "unique_key": ["order_id"], + "snapshot_strategy": "check", + "check_columns": ["*"], + "contract": "enforced", + "columns": {"order_id": None, "status": None}, + }, + (_ORDERS,), + ), + ({"materialized": "ledger", "placeholders": {"region": "emea"}}, ()), + ( + {"materialized": "table", "migrate_from": "legacy_orders", "old_name_view": "7d"}, + (_ORDERS,), + ), +) +_VALIDATION_POOLS: dict[str, tuple[object, ...]] = { + "materialized": ("table", "view", "incremental", "snapshot", "ledger", "archive", 5), + "incremental_strategy": ("append", "delete_insert", "merge", "upsert", 3, _ABSENT), + "cursor": ("updated_at", "order_id", 7, _ABSENT), + "cursor_type": ("timestamp", "integer", "date", _ABSENT), + "cursor_grain": ("day", "hour", "month", "week", _ABSENT), + "cursor_start": ( + "2024-01-01", + "2024-02-01T00:00:00", + "2024-01-01 10:00:00+02:00", + "2024-01-01T00:00:00.123Z", + "2024-13-01", + "20240101", + "2024-W01-1", + "0001-01-01T00:00:00+01:00", + 10, + "15", + "1e3", + "abc", + date(2024, 1, 1), + True, + _ABSENT, + ), + "cursor_end": ( + "2024-01-01", + "2024-06-01 00:00", + 5, + 40, + "30", + "2025-01-01T00:00:00-05:00", + "9999-12-31T23:59:00-01:00", + ), + "unique_key": ("order_id", ["order_id", "customer_id"], ["\udcff"], [], (), 5, _ABSENT), + "merge_exclude_columns": (["amount"], ["ORDER_ID"], ["a", "A"], [""], "amount", _ABSENT), + "incremental_mode": ("full", "microbatch", "batch", _ABSENT), + "microbatch_strategy": ("rolling_window", "watermark", "hourly", _ABSENT), + "cursor_watermark_mode": ("all", "any", "none", _ABSENT), + "batch_size": ("1d", "effective", "2h", "1mo", 5, _ABSENT), + "batch_concurrency": (1, 2, 0, -1, True, "2", 2**70, _ABSENT), + "unaccounted_partition_policy": ("synthesize", "recover_all", "skip", _ABSENT), + "cursor_inputs": ( + {"orders": "updated_at"}, + _WATERMARK_INPUTS, + {"customers": {"column": "c", "roles": ["watermark"]}}, + {"orders": {"column": "c", "roles": ["filter", "filter"]}}, + {}, + {"orders": ""}, + {"invoices": "updated_at"}, + ["orders"], + _ABSENT, + ), + "max_microbatches": (1, 3, 10, 0, True, "3", _ABSENT), + "microbatch_limit": ( + {"max_batches": 5, "action": "cap_from_start"}, + {"max_batches": 2, "action": "warn"}, + {"max_batches": 0, "action": "error"}, + {"max_batches": 5}, + _ABSENT, + ), + "lookback": ("3d", "1mo", "2h", "1mo2d", "soon", _ABSENT), + "on_schema_change": ("fail", "ignore", "drop", 5, _ABSENT), + "replay_on_change": ("forward", "full", "bounded-14d", "bounded- 1mo ", "bounded-x", _ABSENT), + "append_cursor_inclusive": (True, False, "yes", _ABSENT), + "full_refresh": (True, False, "no", _ABSENT), + "cursor_start_max_ahead": ("disabled", "0d", "7d", "soon", _ABSENT), + "cursor_start_max_action": ("cap", "error", "warn", _ABSENT), + "cursor_future_max_distance": ("1d", "disabled", "later", _ABSENT), + "cursor_future_action": ("cap", "error", ["cap"], _ABSENT), + "contract": ("enforced", "none", "strict", 1, _ABSENT), + "columns": ({"order_id": None, "updated_at": None}, _ABSENT), + "snapshot_strategy": ("timestamp", "check", "hybrid", _ABSENT), + "updated_at": ("updated_at", "status", 5, _ABSENT), + "observed_at": ("loaded_at", _ABSENT), + "historical_input": ("snapshot", "changes", "events", _ABSENT), + "initial_valid_from": ("updated_at", "observed_at", "execution_time", "now", _ABSENT), + "snapshot_full_refresh": ("deny", "allow", "always", _ABSENT), + "snapshot_schema_change": ("deny", "append_new_columns", "drop", _ABSENT), + "check_columns": (["*"], ["*", "status"], ["status"], ["\udcff"], [], ("*",), _ABSENT), + "invalidate_hard_deletes": (True, False, "yes", _ABSENT), + "valid_from_column": ("valid_from", "VALID_TO", _ABSENT), + "valid_to_column": ("valid_to", _ABSENT), + "migrate_from": ("legacy_orders", " ", "orders_daily", " ORDERS_DAILY ", 5, _ABSENT), + "migrate_force": (True, False, "yes", _ABSENT), + "old_name_view": ("7d", " 2d ", False, True, "soon", None, _ABSENT), + "placeholders": ({"region": "emea"}, {"other": "x"}, ["region"], _ABSENT), +} +_RETENTIONS: tuple[ResolvedTimeTravelRetention, ...] = ( + ResolvedTimeTravelRetention(), + ResolvedTimeTravelRetention(), + ResolvedTimeTravelRetention(desired_days=7, unmanaged=False), +) +_TABLE_TYPES: tuple[ResolvedTableType, ...] = ( + ResolvedTableType(), + ResolvedTableType(), + ResolvedTableType(value=TableType.PERMANENT, source=TableTypeSource.MODEL, declared=True), +) +_DECLARED_COLUMNS: tuple[tuple[SchemaColumn, ...] | None, ...] = ( + None, + None, + (SchemaColumn(name="order_id"), SchemaColumn(name="updated_at")), + (SchemaColumn(name="customer_id"),), +) +_QUERIES: tuple[str, ...] = ( + "SELECT 1", + "SELECT 1", + "SELECT '@@@region' AS marker, @@@region AS region", + "SELECT @@@other", + "SELECT '@@@' AS marker", +) + + +@dataclass(frozen=True) +class ModelValidationParity: + """How native validation compared with the Python validators over one corpus.""" + + mismatches: list[dict[str, object]] + native_accepted: int + python_accepted: int + python_rejected: int + + +@dataclass(frozen=True) +class ConfigBuildCase: + """One project's layers and target and one model file to build config for.""" + + project_config: ProjectConfig + target_config: TargetConfig | None + effective_target_name: str | None + model_file: DiscoveredSqlModelFile + + +@dataclass(frozen=True) +class ModelConfigBuildParity: + """How native config builds compared with Python's over one corpus.""" + + mismatches: list[tuple[object, object, object]] + built: int + deferred: int + python_raised: int + + +def generated_validation_requests( + *, rng: random.Random, count: int +) -> list[ModelValidationRequest]: + """Return seeded effective configs mixing valid profiles with invalid and unusual values.""" + + return [_validation_request(rng=rng) for _ in range(count)] + + +def model_validation_parity(*, requests: list[ModelValidationRequest]) -> ModelValidationParity: + """Validate natively and in Python; native may reject valid configs but never accept invalid.""" + + session: NativeModelConfigSession = _session( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + target_config=None, + effective_target_name=None, + ) + native: list[bool] = [ + native_model_validators_accept(session=session, request=request) for request in requests + ] + python: list[bool] = [_python_accepts(request) for request in requests] + wrongly_accepted: list[bool] = [ + accepted and not valid for accepted, valid in zip(native, python, strict=True) + ] + return ModelValidationParity( + mismatches=[request.config.values for request in compress(requests, wrongly_accepted)], + native_accepted=sum(native), + python_accepted=sum(python), + python_rejected=len(python) - sum(python), + ) + + +def validation_outcomes(*, values: dict[str, object]) -> tuple[bool, str]: + """Return whether native validation accepts `values` and the Python validators' error.""" + + request: ModelValidationRequest = ModelValidationRequest( + model_file=_model_file(relative_path="marts/orders_daily.sql", header_values={}), + config=CompileModelConfig(values=values), + references=(_ORDERS,), + declared_columns=None, + query_sql="SELECT 1", + ) + session: NativeModelConfigSession = _session( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + target_config=None, + effective_target_name=None, + ) + return ( + native_model_validators_accept(session=session, request=request), + _python_error(request), + ) + + +def _python_error(request: ModelValidationRequest) -> str: + try: + run_python_model_validators(context=_VALIDATOR_CONTEXT, request=request) + except Exception as error: # noqa: BLE001 - the exact Python outcome, whatever it is + return f"{type(error).__name__}: {error}" + return "accepted" + + +def _validation_request(*, rng: random.Random) -> ModelValidationRequest: + profile, references = rng.choice(_PROFILES) + values: dict[str, object] = dict(profile) + mutated: list[str] = rng.sample(sorted(_VALIDATION_POOLS), k=rng.choice((0, 0, 1, 1, 2, 3))) + values.update((key, rng.choice(_VALIDATION_POOLS[key])) for key in mutated) + return ModelValidationRequest( + model_file=_model_file(relative_path="marts/orders_daily.sql", header_values={}), + config=CompileModelConfig( + values=dict( + compress(values.items(), [value is not _ABSENT for value in values.values()]) + ), + time_travel_retention=rng.choice(_RETENTIONS), + table_type=rng.choice(_TABLE_TYPES), + ), + references=rng.choice( + (references, references, references, tuple(rng.sample(_REFERENCES, k=2))) + ), + declared_columns=rng.choice(_DECLARED_COLUMNS), + query_sql=rng.choice(_QUERIES), + ) + + +def _python_accepts(request: ModelValidationRequest) -> bool: + try: + run_python_model_validators(context=_VALIDATOR_CONTEXT, request=request) + except Exception: # noqa: BLE001 - any Python failure means native must not accept + return False + return True + + +_DEFAULTS: tuple[DefaultsConfig, ...] = ( + DefaultsConfig(), + DefaultsConfig(materialized="table", schema="analytics", tags=("core",)), + DefaultsConfig( + materialized="view", + database="warehouse", + schema="${region}_core", + full_refresh=True, + row_diff_exclude_columns=("loaded_at",), + row_diff_tolerances={"by_type": {"DOUBLE": 0.1}, "sample": 3}, + ), + DefaultsConfig(pre_hooks=[SqlHookEntry(statement="SELECT 1")], tags=("core", "daily")), + DefaultsConfig(incremental_strategy="append", cursor_start="2024-01-01", contract="none"), +) +_PATH_DEFAULTS: tuple[dict[str, dict[str, object]], ...] = ( + {}, + {"staging": {"materialized": "view", "tags": ["staging", "core"]}}, + { + "staging/*": {"schema": "${CTX:model.name}_stg", "row_diff_exclude_columns": ["x"]}, + "marts": { + "materialized": "table", + "row_diff_tolerances": {"by_column": {"amount": 0.5}, "by_type": "loose"}, + "post_hooks": [NamedSqlHookEntry(name="audit_refresh", kwargs={})], + }, + }, + { + "**/finance": {"database": "${ENV:SQB_BUILD_DB}", "tags": "finance"}, + "marts/**": {"tags": [1, 2], "schema": "${CTX:run.target}"}, + }, + {"*/orders.sql": {"materialized": "table"}, "staging/*": {"materialized": "view"}}, +) +_HEADER_POOLS: dict[str, tuple[object, ...]] = { + "materialized": ("table", "incremental", "view", "snapshot", 5), + "schema": ( + "orders", + "${region}_${CTX:model.name}", + "${CTX:run.target}", + "${ENV:SQB_BUILD_SCHEMA}", + "${missing}", + "${CTX:destination.table}", + "${coalesce(ENV:SQB_UNSET_SCHEMA, 'fallback')}", + ), + "database": ("warehouse", "${ENV:SQB_BUILD_DB}", None), + "alias": ("orders_v2", "${CTX:model.schema}_x", "@cents(amount)"), + "tags": (["daily"], ["core", "daily"], "daily", [1]), + "full_refresh": (True, False), + "time_travel_retention": ("7d", "inherit", "disabled", "0d", "6h", 7), + "table_type": ("permanent", "transient", "inherit", "temporary"), + "row_diff_exclude_columns": (["updated_at"], ("x", "y")), + "row_diff_tolerances": ({"by_type": {"INTEGER": 1}}, {"by_column": {"a": 2}, "x": 1}, "loose"), + "description": ("Orders.", "Totals for @cents(1)"), + "unique_key": (["order_id"],), + "pre_hooks": ([SqlHookEntry(statement="SELECT 1")], ["SELECT 1"], None), + "post_hooks": ([PythonHookEntry(name="notify", kwargs={})],), +} +_TARGETS: tuple[TargetConfig | None, ...] = ( + None, + TargetConfig(), + TargetConfig(database="dev_db", schema="${CTX:model.schema}_dev"), + TargetConfig(schema="preserve"), + TargetConfig(database="preserve", schema="analytics"), + TargetConfig( + schema="${ENV:SQB_BUILD_SCHEMA}", + default_table_type=TableType.PERMANENT, + time_travel_retention=AuthoredTimeTravelRetention(desired_days=3), + time_travel_retention_by_materialization={ + "incremental": AuthoredTimeTravelRetention(unmanaged=True) + }, + ), +) +_MATERIALIZATION_DEFAULTS: tuple[MaterializationDefaultsConfig, ...] = ( + MaterializationDefaultsConfig(), + MaterializationDefaultsConfig( + table=MaterializationRetentionDefaults( + time_travel_retention=AuthoredTimeTravelRetention(desired_days=1), + table_type=TableType.PERMANENT, + ) + ), +) +_MODEL_PATHS: tuple[str, ...] = ( + "staging/orders.sql", + "marts/finance/revenue.sql", + "orders.sql", +) + + +def generated_config_builds(*, rng: random.Random, count: int) -> list[ConfigBuildCase]: + """Return seeded project layers, targets and MODEL headers, valid and invalid.""" + + return [_config_build_case(rng=rng) for _ in range(count)] + + +def model_config_build_parity(*, cases: list[ConfigBuildCase]) -> ModelConfigBuildParity: + """Build natively and in Python; native may defer but never differ or hide an error.""" + + outcomes: list[tuple[object, object, object]] = [ + (case.model_file.header_values, _python_build(case), _native_build(case)) for case in cases + ] + return ModelConfigBuildParity( + mismatches=list( + compress( + outcomes, + [native not in (_DEFERRED, python) for _, python, native in outcomes], + ) + ), + built=sum(native not in (_DEFERRED, _RAISES) for _, _, native in outcomes), + deferred=sum(native == _DEFERRED for _, _, native in outcomes), + python_raised=sum(python == _RAISES for _, python, _ in outcomes), + ) + + +def _config_build_case(*, rng: random.Random) -> ConfigBuildCase: + keys: list[str] = rng.sample(sorted(_HEADER_POOLS), k=rng.choice((0, 1, 2, 3, 4))) + return ConfigBuildCase( + project_config=ProjectConfig( + name="orders", + adapter="duckdb", + defaults=rng.choice(_DEFAULTS), + path_defaults=rng.choice(_PATH_DEFAULTS), + materialization_defaults=rng.choice(_MATERIALIZATION_DEFAULTS), + ), + target_config=rng.choice(_TARGETS), + effective_target_name=rng.choice(("dev", None)), + model_file=_model_file( + relative_path=rng.choice(_MODEL_PATHS), + header_values={key: rng.choice(_HEADER_POOLS[key]) for key in keys}, + ), + ) + + +def _python_build(case: ConfigBuildCase) -> object: + with COMPILE_INPUT_READS.recording() as reads: + try: + matched: str | None = find_matching_path_default( + model_file=case.model_file, path_defaults=case.project_config.path_defaults + ) + config: CompileModelConfig = build_model_config( + request=ModelConfigBuildRequest( + defaults=case.project_config.defaults, + path_defaults=case.project_config.path_defaults, + matched_path_default=matched, + model_header_values=case.model_file.header_values, + effective_vars={"region": "emea"}, + target_config=case.target_config, + model_name=case.model_file.file_path.stem, + effective_target_name=case.effective_target_name, + run_id=_RUN_ID, + materialization_defaults=case.project_config.materialization_defaults, + ) + ) + except Exception: # noqa: BLE001 - every Python failure must stop the native build too + return _RAISES + return (_config_shape(config), reads.environment_names, reads.read_run_id) + + +def _native_build(case: ConfigBuildCase) -> object: + session: NativeModelConfigSession = _session( + project_config=case.project_config, + target_config=case.target_config, + effective_target_name=case.effective_target_name, + ) + with COMPILE_INPUT_READS.recording() as reads: + try: + matched: str | None = native_path_default(session=session, model_file=case.model_file) + except Exception: # noqa: BLE001 - path conflicts are reported by Python + return _RAISES + config: CompileModelConfig | None = build_native_model_config( + session=session, model_file=case.model_file, matched_path_default=matched + ) + shapes: list[object] = [ + (_config_shape(built), reads.environment_names, reads.read_run_id) + for built in cast(list[CompileModelConfig], list(compress([config], [config is not None]))) + ] + return (*shapes, _DEFERRED)[0] + + +def _config_shape(config: CompileModelConfig) -> tuple[object, ...]: + items: list[tuple[str, object]] = list(config.values.items()) + hooks: list[bool] = [key in _HOOK_KEYS for key, _ in items] + return ( + list(compress(items, [not hook for hook in hooks])), + sorted(compress(items, hooks), key=itemgetter(0)), + replace(config, values={}), + ) + + +def _session( + *, + project_config: ProjectConfig, + target_config: TargetConfig | None, + effective_target_name: str | None, +) -> NativeModelConfigSession: + return native_model_config_session( + inputs=NativeModelConfigInputs( + project_config=project_config, + target_config=target_config, + effective_vars={"region": "emea"}, + effective_target_name=effective_target_name, + run_id=_RUN_ID, + microbatch_concurrency=False, + ), + names=_NAMES, + ) + + +def _model_file(*, relative_path: str, header_values: dict[str, object]) -> DiscoveredSqlModelFile: + return DiscoveredSqlModelFile( + file_path=Path("/project/models") / relative_path, + relative_path=Path("models") / relative_path, + contents="", + header_values=header_values, + header_column_locations={}, + output_column_locations={}, + query_sql="SELECT 1", + ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py new file mode 100644 index 0000000000..2d141ae4e5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py @@ -0,0 +1,178 @@ +"""Native model config builds and validator checks match Python exactly or defer to it.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.compiler.compile.native_model_config._test_types import ( + ModelConfigBuildParityTestCase, + ModelValidationParityTestCase, + NativeRejectionParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.compile.native_model_config.helpers import ( + BUILD_ENVIRONMENT, + ModelConfigBuildParity, + ModelValidationParity, + generated_config_builds, + generated_validation_requests, + model_config_build_parity, + model_validation_parity, + validation_outcomes, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelValidationParityTestCase( + description="seeded profiles with invalid and unusual values", + seed=20261008, + count=6000, + expected_minimum_native_accepted=1500, + expected_minimum_python_rejected=1500, + expected_minimum_acceptance_percent=90, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_configs_when_validating_then_native_accepts_only_what_python_accepts( + test_case: ModelValidationParityTestCase, +) -> None: + parity: ModelValidationParity = model_validation_parity( + requests=generated_validation_requests( + rng=random.Random(test_case.seed), count=test_case.count + ) + ) + + assert ( + parity.mismatches, + parity.native_accepted >= test_case.expected_minimum_native_accepted, + parity.python_rejected >= test_case.expected_minimum_python_rejected, + parity.native_accepted * 100 + >= parity.python_accepted * test_case.expected_minimum_acceptance_percent, + ) == ([], True, True, True), ( + parity.native_accepted, + parity.python_accepted, + parity.python_rejected, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelConfigBuildParityTestCase( + description="seeded defaults, path defaults, headers and targets", + seed=20261009, + count=6000, + expected_minimum_built=2000, + expected_minimum_python_raised=500, + expected_minimum_build_percent=90, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_layers_when_building_config_then_native_matches_python_or_defers( + test_case: ModelConfigBuildParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + for name, value in BUILD_ENVIRONMENT.items(): + monkeypatch.setenv(name, value) + + parity: ModelConfigBuildParity = model_config_build_parity( + cases=generated_config_builds(rng=random.Random(test_case.seed), count=test_case.count) + ) + + assert ( + parity.mismatches[:3], + parity.built >= test_case.expected_minimum_built, + parity.python_raised >= test_case.expected_minimum_python_raised, + parity.built * 100 + >= (test_case.count - parity.python_raised) * test_case.expected_minimum_build_percent, + ) == ([], True, True, True), (parity.built, parity.deferred, parity.python_raised) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeRejectionParityTestCase( + description="a start bound shifted before year 1 in UTC", + values={ + "materialized": "incremental", + "incremental_strategy": "append", + "cursor": "updated_at", + "cursor_type": "timestamp", + "cursor_grain": "day", + "cursor_start": "0001-01-01T00:00:00+01:00", + "cursor_end": "2024-01-01", + }, + expected_native_accepted=False, + expected_python_error=( + "CompileInputError: model 'orders_daily': cursor_start value " + "'0001-01-01T00:00:00+01:00' falls outside years 1-9999 once converted to UTC" + ), + ), + NativeRejectionParityTestCase( + description="an end bound shifted past year 9999 in UTC", + values={ + "materialized": "incremental", + "incremental_strategy": "append", + "cursor": "updated_at", + "cursor_type": "timestamp", + "cursor_grain": "day", + "cursor_start": "2024-01-01", + "cursor_end": "9999-12-31T23:59:00-01:00", + }, + expected_native_accepted=False, + expected_python_error=( + "CompileInputError: model 'orders_daily': cursor_end value " + "'9999-12-31T23:59:00-01:00' falls outside years 1-9999 once converted to UTC" + ), + ), + NativeRejectionParityTestCase( + description="an unreadable unique key under an enforced contract", + values={ + "materialized": "incremental", + "incremental_strategy": "merge", + "unique_key": ["\udcff"], + "contract": "enforced", + "columns": {"order_id": None}, + }, + expected_native_accepted=False, + expected_python_error=( + "CompileInputError: model 'orders_daily': unique_key references column " + "'\udcff' not declared in enforced contract" + ), + ), + NativeRejectionParityTestCase( + description="an unreadable snapshot check column under an enforced contract", + values={ + "materialized": "snapshot", + "unique_key": "order_id", + "snapshot_strategy": "check", + "check_columns": ["\udcff"], + "contract": "enforced", + "columns": {"order_id": None}, + }, + expected_native_accepted=False, + expected_python_error=( + "CompileInputError: model 'orders_daily': check_columns references column " + "'\udcff' not declared in enforced contract" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_rejected_edge_configs_when_validating_then_native_defers_to_python( + test_case: NativeRejectionParityTestCase, +) -> None: + accepted, python_error = validation_outcomes(values=test_case.values) + + assert (accepted, python_error) == ( + test_case.expected_native_accepted, + test_case.expected_python_error, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 65aa6aa58f..a778c0f5a7 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -177,9 +177,9 @@ def test_given_generated_templates_when_expanding_then_native_matches_python_or_ engine="native-preview", expected_native_calls={ "parse_model_header_metadata": 1, - "expand_config_templates": 7, - "config_contains_template": 2, - "config_contains_macro_call": 2, + "expand_config_templates": 1, + "config_contains_template": 0, + "config_contains_macro_call": 0, }, ), ],