Skip to content

Commit 24e47aa

Browse files
authored
feat(native): build model config and validate models natively behind the preview engine (#1034)
* feat(native): build model config and validate models natively behind the preview engine * fix(native): defer UTC-overflowing cursor bounds and unreadable column names to Python
1 parent 7afa795 commit 24e47aa

58 files changed

Lines changed: 4991 additions & 79 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
7474
tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \
7575
tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \
7676
tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \
77+
tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_engines.py \
7778
tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \
7879
tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \
7980
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_hooks.py \

‎crates/sqlbuild-model-config/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
pub mod config_presence;
66
pub mod header_metadata;
7+
pub mod model_validation;
8+
pub mod path_defaults;
79
pub mod templates;
810
pub mod types;
911

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
//! Effective config values read the way the Python validators read the dict.
2+
3+
use std::collections::HashMap;
4+
5+
use crate::model_validation::_helpers::text::{python_lower, python_strip};
6+
use crate::model_validation::models::Rejected;
7+
use crate::types::{AuthoredNode, NodeKind};
8+
9+
/// Config values by string key; Python's `dict.get` never matches other keys.
10+
pub(crate) struct ConfigView<N> {
11+
values: HashMap<String, N>,
12+
}
13+
14+
impl<N: AuthoredNode> ConfigView<N> {
15+
/// View the entries of one config mapping.
16+
pub(crate) fn new(entries: Vec<(N, N)>) -> Self {
17+
let mut values: HashMap<String, N> = HashMap::with_capacity(entries.len());
18+
for (key, value) in entries {
19+
if let Some(text) = key.text() {
20+
values.insert(text, value);
21+
}
22+
}
23+
Self { values }
24+
}
25+
26+
/// Return whether `key in values`, even when its value is `None`.
27+
pub(crate) fn contains_key(&self, key: &str) -> bool {
28+
self.values.contains_key(key)
29+
}
30+
31+
/// Return `values.get(key)` unless it is absent or `None`.
32+
pub(crate) fn get(&self, key: &str) -> Option<&N> {
33+
self.values
34+
.get(key)
35+
.filter(|value| value.kind() != NodeKind::Null)
36+
}
37+
38+
/// Return whether `values.get(key) is not None`.
39+
pub(crate) fn present(&self, key: &str) -> bool {
40+
self.get(key).is_some()
41+
}
42+
43+
/// Return whether any of `keys` is set.
44+
pub(crate) fn any_present(&self, keys: &[&str]) -> bool {
45+
keys.iter().any(|key| self.present(key))
46+
}
47+
48+
/// Read `get_config_str`: a string, `None`, or a rejection for any other type.
49+
pub(crate) fn string(&self, key: &str) -> Result<Option<String>, Rejected> {
50+
self.get(key).map(string_value).transpose()
51+
}
52+
53+
/// Return whether the value equals `text`, as Python's `==` against a string.
54+
pub(crate) fn equals(&self, key: &str, text: &str) -> bool {
55+
self.get(key).is_some_and(|value| value.is_text(text))
56+
}
57+
}
58+
59+
/// Return the text of a string value, or reject any other type.
60+
pub(crate) fn string_value<N: AuthoredNode>(value: &N) -> Result<String, Rejected> {
61+
if value.kind() == NodeKind::Str {
62+
value.text().ok_or(Rejected)
63+
} else {
64+
Err(Rejected)
65+
}
66+
}
67+
68+
/// Return `None` for an absent or `None` value, the text of a string, or reject other types.
69+
pub(crate) fn optional_text<N: AuthoredNode>(
70+
value: Option<&N>,
71+
) -> Result<Option<String>, Rejected> {
72+
match value {
73+
Some(node) if node.kind() != NodeKind::Null => string_value(node).map(Some),
74+
_ => Ok(None),
75+
}
76+
}
77+
78+
/// Return whether `text in vocabulary`.
79+
pub(crate) fn one_of(text: &str, vocabulary: &[&str]) -> bool {
80+
vocabulary.contains(&text)
81+
}
82+
83+
/// Return whether an optional string is in `vocabulary`; `None` is not.
84+
pub(crate) fn optional_one_of(text: Option<&str>, vocabulary: &[&str]) -> bool {
85+
text.is_some_and(|value| one_of(value, vocabulary))
86+
}
87+
88+
/// Read `_string_sequence`: a string, or the strings of a list or tuple; reject unreadable text.
89+
pub(crate) fn string_sequence<N: AuthoredNode>(value: Option<&N>) -> Result<Vec<String>, Rejected> {
90+
match value.map(|node| (node.kind(), node)) {
91+
Some((NodeKind::Str, node)) => string_value(node).map(|text| vec![text]),
92+
Some((NodeKind::List | NodeKind::Tuple, node)) => node
93+
.items()
94+
.iter()
95+
.filter(|item| item.kind() == NodeKind::Str)
96+
.map(string_value)
97+
.collect(),
98+
_ => Ok(Vec::new()),
99+
}
100+
}
101+
102+
/// Return whether a value is set and not an empty list or tuple, like `_has_config_value`.
103+
pub(crate) fn has_config_value<N: AuthoredNode>(value: Option<&N>) -> bool {
104+
value.is_some_and(|node| {
105+
!matches!(node.kind(), NodeKind::List | NodeKind::Tuple) || !node.items().is_empty()
106+
})
107+
}
108+
109+
/// Return the text of a string whose Python `strip()` is non-empty, or reject.
110+
pub(crate) fn non_blank_string<N: AuthoredNode>(value: &N) -> Result<String, Rejected> {
111+
let text = string_value(value)?;
112+
if python_strip(&text)?.is_empty() {
113+
Err(Rejected)
114+
} else {
115+
Ok(text)
116+
}
117+
}
118+
119+
/// Reject duplicate names under Python's `str.lower()`.
120+
pub(crate) fn reject_case_insensitive_duplicates(names: &[String]) -> Result<(), Rejected> {
121+
let mut seen = Vec::with_capacity(names.len());
122+
for name in names {
123+
let lowered = python_lower(name)?;
124+
if seen.contains(&lowered) {
125+
return Err(Rejected);
126+
}
127+
seen.push(lowered);
128+
}
129+
Ok(())
130+
}
Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
//! `cursor_start` and `cursor_end` values Python certainly accepts, with their ordering keys.
2+
3+
use crate::model_validation::constants::{
4+
CLOCK_FIELD_LENGTH, CLOCK_PART_COUNTS, CLOCK_PARTS_WITH_SECONDS, INTEGER_CURSOR,
5+
ISO_DATE_LENGTH, ISO_DATE_SEPARATOR_OFFSETS, LAST_SHIFTED_MONTH, MAX_HOUR, MAX_MINUTE,
6+
MAX_YEAR, MIN_YEAR, TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH,
7+
};
8+
use crate::model_validation::models::Rejected;
9+
use crate::types::AuthoredNode;
10+
11+
const MICROS_PER_SECOND: i128 = 1_000_000;
12+
const SECONDS_PER_DAY: i128 = 86_400;
13+
const SECONDS_PER_HOUR: i128 = 3_600;
14+
const SECONDS_PER_MINUTE: i128 = 60;
15+
const CIVIL_EPOCH_OFFSET_DAYS: i128 = 719_468;
16+
const FRACTION_DIGITS: [usize; 2] = [3, 6];
17+
18+
/// Return the `_cursor_contract_key` order key: UTC microseconds, or the integer itself.
19+
pub(crate) fn cursor_bound_key<N: AuthoredNode>(
20+
value: &N,
21+
cursor_type: &str,
22+
) -> Result<i128, Rejected> {
23+
if let Some(number) = value.integer().filter(|_| cursor_type == INTEGER_CURSOR) {
24+
return Ok(i128::from(number));
25+
}
26+
let text = value.text().ok_or(Rejected)?;
27+
match cursor_type {
28+
INTEGER_CURSOR => integer_key(&text),
29+
TIMESTAMP_CURSOR => timestamp_key(&text),
30+
_ => Err(Rejected),
31+
}
32+
}
33+
34+
/// Return whether two timestamp keys keep their order through Python's float timestamps.
35+
pub(crate) fn keys_ordered(start: i128, end: i128, cursor_type: &str) -> bool {
36+
if cursor_type == TIMESTAMP_CURSOR {
37+
end - start >= MICROS_PER_SECOND
38+
} else {
39+
start < end
40+
}
41+
}
42+
43+
fn integer_key(text: &str) -> Result<i128, Rejected> {
44+
let digits = text.strip_prefix(['+', '-']).unwrap_or(text);
45+
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
46+
return Err(Rejected);
47+
}
48+
text.strip_prefix('+')
49+
.unwrap_or(text)
50+
.parse()
51+
.map_err(|_| Rejected)
52+
}
53+
54+
fn timestamp_key(text: &str) -> Result<i128, Rejected> {
55+
if !text.is_ascii() {
56+
return Err(Rejected);
57+
}
58+
let clock_start = ISO_DATE_LENGTH + 1;
59+
let (date, time) = match text.len() {
60+
ISO_DATE_LENGTH => (text, None),
61+
length
62+
if length > clock_start && matches!(text.as_bytes()[ISO_DATE_LENGTH], b'T' | b' ') =>
63+
{
64+
(&text[..ISO_DATE_LENGTH], Some(&text[clock_start..]))
65+
}
66+
_ => return Err(Rejected),
67+
};
68+
let days = date_days(date)?;
69+
let (micros, offset_seconds) = match time {
70+
None => (0, 0),
71+
Some(clock) => clock_micros(clock)?,
72+
};
73+
let utc_micros = (days * SECONDS_PER_DAY - offset_seconds) * MICROS_PER_SECOND + micros;
74+
let day_micros = SECONDS_PER_DAY * MICROS_PER_SECOND;
75+
let first_micros = days_from_civil(MIN_YEAR, 1, 1) * day_micros;
76+
let end_micros = days_from_civil(MAX_YEAR + 1, 1, 1) * day_micros;
77+
if utc_micros < first_micros || utc_micros >= end_micros {
78+
return Err(Rejected);
79+
}
80+
Ok(utc_micros)
81+
}
82+
83+
fn date_days(date: &str) -> Result<i128, Rejected> {
84+
let bytes = date.as_bytes();
85+
let [year_end, month_end] = ISO_DATE_SEPARATOR_OFFSETS;
86+
if bytes[year_end] != b'-' || bytes[month_end] != b'-' {
87+
return Err(Rejected);
88+
}
89+
let year = number(&date[..year_end])?;
90+
let month = number(&date[year_end + 1..month_end])?;
91+
let day = number(&date[month_end + 1..])?;
92+
if year == 0 || !(1..=12).contains(&month) || day == 0 || day > month_days(year, month) {
93+
return Err(Rejected);
94+
}
95+
Ok(days_from_civil(year, month, day))
96+
}
97+
98+
fn clock_micros(clock: &str) -> Result<(i128, i128), Rejected> {
99+
let (clock, offset_seconds) = split_offset(clock)?;
100+
let (hms, fraction) = clock
101+
.split_once('.')
102+
.map_or((clock, None), |(hms, fraction)| (hms, Some(fraction)));
103+
let parts: Vec<&str> = hms.split(':').collect();
104+
if !CLOCK_PART_COUNTS.contains(&parts.len())
105+
|| parts.iter().any(|part| part.len() != CLOCK_FIELD_LENGTH)
106+
|| (fraction.is_some() && parts.len() != CLOCK_PARTS_WITH_SECONDS)
107+
{
108+
return Err(Rejected);
109+
}
110+
let hour = number(parts[0])?;
111+
let minute = number(parts[1])?;
112+
let second = parts.get(2).map_or(Ok(0), |part| number(part))?;
113+
if hour > MAX_HOUR || minute > MAX_MINUTE || second > MAX_MINUTE {
114+
return Err(Rejected);
115+
}
116+
let fraction_micros = match fraction {
117+
None => 0,
118+
Some(digits) if FRACTION_DIGITS.contains(&digits.len()) => {
119+
number(digits)? * 10_i128.pow(6 - u32::try_from(digits.len()).map_err(|_| Rejected)?)
120+
}
121+
Some(_) => return Err(Rejected),
122+
};
123+
let seconds = hour * SECONDS_PER_HOUR + minute * SECONDS_PER_MINUTE + second;
124+
Ok((
125+
seconds * MICROS_PER_SECOND + fraction_micros,
126+
offset_seconds,
127+
))
128+
}
129+
130+
fn split_offset(clock: &str) -> Result<(&str, i128), Rejected> {
131+
if let Some(clock) = clock.strip_suffix('Z') {
132+
return Ok((clock, 0));
133+
}
134+
let Some(sign_index) = clock.rfind(['+', '-']) else {
135+
return Ok((clock, 0));
136+
};
137+
let offset = &clock[sign_index + 1..];
138+
let bytes = offset.as_bytes();
139+
if offset.len() != UTC_OFFSET_LENGTH || bytes[CLOCK_FIELD_LENGTH] != b':' {
140+
return Err(Rejected);
141+
}
142+
let hours = number(&offset[..CLOCK_FIELD_LENGTH])?;
143+
let minutes = number(&offset[CLOCK_FIELD_LENGTH + 1..])?;
144+
if hours > MAX_HOUR || minutes > MAX_MINUTE {
145+
return Err(Rejected);
146+
}
147+
let magnitude = hours * SECONDS_PER_HOUR + minutes * SECONDS_PER_MINUTE;
148+
let sign = if clock.as_bytes()[sign_index] == b'-' {
149+
-1
150+
} else {
151+
1
152+
};
153+
Ok((&clock[..sign_index], sign * magnitude))
154+
}
155+
156+
fn number(digits: &str) -> Result<i128, Rejected> {
157+
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
158+
return Err(Rejected);
159+
}
160+
digits.parse().map_err(|_| Rejected)
161+
}
162+
163+
fn month_days(year: i128, month: i128) -> i128 {
164+
match month {
165+
2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29,
166+
2 => 28,
167+
4 | 6 | 9 | 11 => 30,
168+
_ => 31,
169+
}
170+
}
171+
172+
fn days_from_civil(year: i128, month: i128, day: i128) -> i128 {
173+
let shifted_year = if month <= LAST_SHIFTED_MONTH {
174+
year - 1
175+
} else {
176+
year
177+
};
178+
let era = shifted_year.div_euclid(400);
179+
let year_of_era = shifted_year - era * 400;
180+
let shifted_month = (month + 9) % 12;
181+
let day_of_year = (153 * shifted_month + 2) / 5 + day - 1;
182+
let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
183+
era * 146_097 + day_of_era - CIVIL_EPOCH_OFFSET_DAYS
184+
}

0 commit comments

Comments
 (0)