feat(native): build model config and validate models natively behind the preview engine

Makefile

@@ -74,6 +74,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \

crates/sqlbuild-model-config/src/lib.rs

@@ -4,6 +4,8 @@
 
 pub mod config_presence;
 pub mod header_metadata;
+pub mod model_validation;
+pub mod path_defaults;
 pub mod templates;
 pub mod types;
 

crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs

@@ -0,0 +1,130 @@
+//! Effective config values read the way the Python validators read the dict.
+
+use std::collections::HashMap;
+
+use crate::model_validation::_helpers::text::{python_lower, python_strip};
+use crate::model_validation::models::Rejected;
+use crate::types::{AuthoredNode, NodeKind};
+
+/// Config values by string key; Python's `dict.get` never matches other keys.
+pub(crate) struct ConfigView<N> {
+    values: HashMap<String, N>,
+}
+
+impl<N: AuthoredNode> ConfigView<N> {
+    /// View the entries of one config mapping.
+    pub(crate) fn new(entries: Vec<(N, N)>) -> Self {
+        let mut values: HashMap<String, N> = HashMap::with_capacity(entries.len());
+        for (key, value) in entries {
+            if let Some(text) = key.text() {
+                values.insert(text, value);
+            }
+        }
+        Self { values }
+    }
+
+    /// Return whether `key in values`, even when its value is `None`.
+    pub(crate) fn contains_key(&self, key: &str) -> bool {
+        self.values.contains_key(key)
+    }
+
+    /// Return `values.get(key)` unless it is absent or `None`.
+    pub(crate) fn get(&self, key: &str) -> Option<&N> {
+        self.values
+            .get(key)
+            .filter(|value| value.kind() != NodeKind::Null)
+    }
+
+    /// Return whether `values.get(key) is not None`.
+    pub(crate) fn present(&self, key: &str) -> bool {
+        self.get(key).is_some()
+    }
+
+    /// Return whether any of `keys` is set.
+    pub(crate) fn any_present(&self, keys: &[&str]) -> bool {
+        keys.iter().any(|key| self.present(key))
+    }
+
+    /// Read `get_config_str`: a string, `None`, or a rejection for any other type.
+    pub(crate) fn string(&self, key: &str) -> Result<Option<String>, Rejected> {
+        self.get(key).map(string_value).transpose()
+    }
+
+    /// Return whether the value equals `text`, as Python's `==` against a string.
+    pub(crate) fn equals(&self, key: &str, text: &str) -> bool {
+        self.get(key).is_some_and(|value| value.is_text(text))
+    }
+}
+
+/// Return the text of a string value, or reject any other type.
+pub(crate) fn string_value<N: AuthoredNode>(value: &N) -> Result<String, Rejected> {
+    if value.kind() == NodeKind::Str {
+        value.text().ok_or(Rejected)
+    } else {
+        Err(Rejected)
+    }
+}
+
+/// Return `None` for an absent or `None` value, the text of a string, or reject other types.
+pub(crate) fn optional_text<N: AuthoredNode>(
+    value: Option<&N>,
+) -> Result<Option<String>, Rejected> {
+    match value {
+        Some(node) if node.kind() != NodeKind::Null => string_value(node).map(Some),
+        _ => Ok(None),
+    }
+}
+
+/// Return whether `text in vocabulary`.
+pub(crate) fn one_of(text: &str, vocabulary: &[&str]) -> bool {
+    vocabulary.contains(&text)
+}
+
+/// Return whether an optional string is in `vocabulary`; `None` is not.
+pub(crate) fn optional_one_of(text: Option<&str>, vocabulary: &[&str]) -> bool {
+    text.is_some_and(|value| one_of(value, vocabulary))
+}
+
+/// Read `_string_sequence`: a string, or the strings of a list or tuple; reject unreadable text.
+pub(crate) fn string_sequence<N: AuthoredNode>(value: Option<&N>) -> Result<Vec<String>, Rejected> {
+    match value.map(|node| (node.kind(), node)) {
+        Some((NodeKind::Str, node)) => string_value(node).map(|text| vec![text]),
+        Some((NodeKind::List | NodeKind::Tuple, node)) => node
+            .items()
+            .iter()
+            .filter(|item| item.kind() == NodeKind::Str)
+            .map(string_value)
+            .collect(),
+        _ => Ok(Vec::new()),
+    }
+}
+
+/// Return whether a value is set and not an empty list or tuple, like `_has_config_value`.
+pub(crate) fn has_config_value<N: AuthoredNode>(value: Option<&N>) -> bool {
+    value.is_some_and(|node| {
+        !matches!(node.kind(), NodeKind::List | NodeKind::Tuple) || !node.items().is_empty()
+    })
+}
+
+/// Return the text of a string whose Python `strip()` is non-empty, or reject.
+pub(crate) fn non_blank_string<N: AuthoredNode>(value: &N) -> Result<String, Rejected> {
+    let text = string_value(value)?;
+    if python_strip(&text)?.is_empty() {
+        Err(Rejected)
+    } else {
+        Ok(text)
+    }
+}
+
+/// Reject duplicate names under Python's `str.lower()`.
+pub(crate) fn reject_case_insensitive_duplicates(names: &[String]) -> Result<(), Rejected> {
+    let mut seen = Vec::with_capacity(names.len());
+    for name in names {
+        let lowered = python_lower(name)?;
+        if seen.contains(&lowered) {
+            return Err(Rejected);
+        }
+        seen.push(lowered);
+    }
+    Ok(())
+}

crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs

@@ -0,0 +1,184 @@
+//! `cursor_start` and `cursor_end` values Python certainly accepts, with their ordering keys.
+
+use crate::model_validation::constants::{
+    CLOCK_FIELD_LENGTH, CLOCK_PART_COUNTS, CLOCK_PARTS_WITH_SECONDS, INTEGER_CURSOR,
+    ISO_DATE_LENGTH, ISO_DATE_SEPARATOR_OFFSETS, LAST_SHIFTED_MONTH, MAX_HOUR, MAX_MINUTE,
+    MAX_YEAR, MIN_YEAR, TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH,
+};
+use crate::model_validation::models::Rejected;
+use crate::types::AuthoredNode;
+
+const MICROS_PER_SECOND: i128 = 1_000_000;
+const SECONDS_PER_DAY: i128 = 86_400;
+const SECONDS_PER_HOUR: i128 = 3_600;
+const SECONDS_PER_MINUTE: i128 = 60;
+const CIVIL_EPOCH_OFFSET_DAYS: i128 = 719_468;
+const FRACTION_DIGITS: [usize; 2] = [3, 6];
+
+/// Return the `_cursor_contract_key` order key: UTC microseconds, or the integer itself.
+pub(crate) fn cursor_bound_key<N: AuthoredNode>(
+    value: &N,
+    cursor_type: &str,
+) -> Result<i128, Rejected> {
+    if let Some(number) = value.integer().filter(|_| cursor_type == INTEGER_CURSOR) {
+        return Ok(i128::from(number));
+    }
+    let text = value.text().ok_or(Rejected)?;
+    match cursor_type {
+        INTEGER_CURSOR => integer_key(&text),
+        TIMESTAMP_CURSOR => timestamp_key(&text),
+        _ => Err(Rejected),
+    }
+}
+
+/// Return whether two timestamp keys keep their order through Python's float timestamps.
+pub(crate) fn keys_ordered(start: i128, end: i128, cursor_type: &str) -> bool {
+    if cursor_type == TIMESTAMP_CURSOR {
+        end - start >= MICROS_PER_SECOND
+    } else {
+        start < end
+    }
+}
+
+fn integer_key(text: &str) -> Result<i128, Rejected> {
+    let digits = text.strip_prefix(['+', '-']).unwrap_or(text);
+    if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
+        return Err(Rejected);
+    }
+    text.strip_prefix('+')
+        .unwrap_or(text)
+        .parse()
+        .map_err(|_| Rejected)
+}
+
+fn timestamp_key(text: &str) -> Result<i128, Rejected> {
+    if !text.is_ascii() {
+        return Err(Rejected);
+    }
+    let clock_start = ISO_DATE_LENGTH + 1;
+    let (date, time) = match text.len() {
+        ISO_DATE_LENGTH => (text, None),
+        length
+            if length > clock_start && matches!(text.as_bytes()[ISO_DATE_LENGTH], b'T' | b' ') =>
+        {
+            (&text[..ISO_DATE_LENGTH], Some(&text[clock_start..]))
+        }
+        _ => return Err(Rejected),
+    };
+    let days = date_days(date)?;
+    let (micros, offset_seconds) = match time {
+        None => (0, 0),
+        Some(clock) => clock_micros(clock)?,
+    };
+    let utc_micros = (days * SECONDS_PER_DAY - offset_seconds) * MICROS_PER_SECOND + micros;
+    let day_micros = SECONDS_PER_DAY * MICROS_PER_SECOND;
+    let first_micros = days_from_civil(MIN_YEAR, 1, 1) * day_micros;
+    let end_micros = days_from_civil(MAX_YEAR + 1, 1, 1) * day_micros;
+    if utc_micros < first_micros || utc_micros >= end_micros {
+        return Err(Rejected);
+    }
+    Ok(utc_micros)
+}
+
+fn date_days(date: &str) -> Result<i128, Rejected> {
+    let bytes = date.as_bytes();
+    let [year_end, month_end] = ISO_DATE_SEPARATOR_OFFSETS;
+    if bytes[year_end] != b'-' || bytes[month_end] != b'-' {
+        return Err(Rejected);
+    }
+    let year = number(&date[..year_end])?;
+    let month = number(&date[year_end + 1..month_end])?;
+    let day = number(&date[month_end + 1..])?;
+    if year == 0 || !(1..=12).contains(&month) || day == 0 || day > month_days(year, month) {
+        return Err(Rejected);
+    }
+    Ok(days_from_civil(year, month, day))
+}
+
+fn clock_micros(clock: &str) -> Result<(i128, i128), Rejected> {
+    let (clock, offset_seconds) = split_offset(clock)?;
+    let (hms, fraction) = clock
+        .split_once('.')
+        .map_or((clock, None), |(hms, fraction)| (hms, Some(fraction)));
+    let parts: Vec<&str> = hms.split(':').collect();
+    if !CLOCK_PART_COUNTS.contains(&parts.len())
+        || parts.iter().any(|part| part.len() != CLOCK_FIELD_LENGTH)
+        || (fraction.is_some() && parts.len() != CLOCK_PARTS_WITH_SECONDS)
+    {
+        return Err(Rejected);
+    }
+    let hour = number(parts[0])?;
+    let minute = number(parts[1])?;
+    let second = parts.get(2).map_or(Ok(0), |part| number(part))?;
+    if hour > MAX_HOUR || minute > MAX_MINUTE || second > MAX_MINUTE {
+        return Err(Rejected);
+    }
+    let fraction_micros = match fraction {
+        None => 0,
+        Some(digits) if FRACTION_DIGITS.contains(&digits.len()) => {
+            number(digits)? * 10_i128.pow(6 - u32::try_from(digits.len()).map_err(|_| Rejected)?)
+        }
+        Some(_) => return Err(Rejected),
+    };
+    let seconds = hour * SECONDS_PER_HOUR + minute * SECONDS_PER_MINUTE + second;
+    Ok((
+        seconds * MICROS_PER_SECOND + fraction_micros,
+        offset_seconds,
+    ))
+}
+
+fn split_offset(clock: &str) -> Result<(&str, i128), Rejected> {
+    if let Some(clock) = clock.strip_suffix('Z') {
+        return Ok((clock, 0));
+    }
+    let Some(sign_index) = clock.rfind(['+', '-']) else {
+        return Ok((clock, 0));
+    };
+    let offset = &clock[sign_index + 1..];
+    let bytes = offset.as_bytes();
+    if offset.len() != UTC_OFFSET_LENGTH || bytes[CLOCK_FIELD_LENGTH] != b':' {
+        return Err(Rejected);
+    }
+    let hours = number(&offset[..CLOCK_FIELD_LENGTH])?;
+    let minutes = number(&offset[CLOCK_FIELD_LENGTH + 1..])?;
+    if hours > MAX_HOUR || minutes > MAX_MINUTE {
+        return Err(Rejected);
+    }
+    let magnitude = hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE;
+    let sign = if clock.as_bytes()[sign_index] == b'-' {
+        -1
+    } else {
+        1
+    };
+    Ok((&clock[..sign_index], sign * magnitude))
+}
+
+fn number(digits: &str) -> Result<i128, Rejected> {
+    if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
+        return Err(Rejected);
+    }
+    digits.parse().map_err(|_| Rejected)
+}
+
+fn month_days(year: i128, month: i128) -> i128 {
+    match month {
+        2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29,
+        2 => 28,
+        4 | 6 | 9 | 11 => 30,
+        _ => 31,
+    }
+}
+
+fn days_from_civil(year: i128, month: i128, day: i128) -> i128 {
+    let shifted_year = if month <= LAST_SHIFTED_MONTH {
+        year - 1
+    } else {
+        year
+    };
+    let era = shifted_year.div_euclid(400);
+    let year_of_era = shifted_year - era * 400;
+    let shifted_month = (month + 9) % 12;
+    let day_of_year = (153 * shifted_month + 2) / 5 + day - 1;
+    let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
+    era * 146_097 + day_of_era - CIVIL_EPOCH_OFFSET_DAYS
+}