Skip to content

Commit abbc715

Browse files
committed
fix(native): defer UTC-overflowing cursor bounds and unreadable column names to Python
1 parent 765fca9 commit abbc715

10 files changed

Lines changed: 211 additions & 22 deletions

File tree

‎crates/sqlbuild-model-config/src/model_validation/_helpers/config.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,17 +85,17 @@ pub(crate) fn optional_one_of(text: Option<&str>, vocabulary: &[&str]) -> bool {
8585
text.is_some_and(|value| one_of(value, vocabulary))
8686
}
8787

88-
/// Read `_string_sequence`: a string, or the strings of a list or tuple.
89-
pub(crate) fn string_sequence<N: AuthoredNode>(value: Option<&N>) -> Vec<String> {
88+
/// Read `_string_sequence`: a string, or the strings of a list or tuple; reject unreadable text.
89+
pub(crate) fn string_sequence<N: AuthoredNode>(value: Option<&N>) -> Result<Vec<String>, Rejected> {
9090
match value.map(|node| (node.kind(), node)) {
91-
Some((NodeKind::Str, node)) => node.text().into_iter().collect(),
91+
Some((NodeKind::Str, node)) => string_value(node).map(|text| vec![text]),
9292
Some((NodeKind::List | NodeKind::Tuple, node)) => node
9393
.items()
9494
.iter()
9595
.filter(|item| item.kind() == NodeKind::Str)
96-
.filter_map(AuthoredNode::text)
96+
.map(string_value)
9797
.collect(),
98-
_ => Vec::new(),
98+
_ => Ok(Vec::new()),
9999
}
100100
}
101101

‎crates/sqlbuild-model-config/src/model_validation/_helpers/cursor_bounds.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use crate::model_validation::constants::{
44
CLOCK_FIELD_LENGTH, CLOCK_PART_COUNTS, CLOCK_PARTS_WITH_SECONDS, INTEGER_CURSOR,
55
ISO_DATE_LENGTH, ISO_DATE_SEPARATOR_OFFSETS, LAST_SHIFTED_MONTH, MAX_HOUR, MAX_MINUTE,
6-
TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH,
6+
MAX_YEAR, MIN_YEAR, TIMESTAMP_CURSOR, UTC_OFFSET_LENGTH,
77
};
88
use crate::model_validation::models::Rejected;
99
use crate::types::AuthoredNode;
@@ -70,7 +70,14 @@ fn timestamp_key(text: &str) -> Result<i128, Rejected> {
7070
None => (0, 0),
7171
Some(clock) => clock_micros(clock)?,
7272
};
73-
Ok((days * SECONDS_PER_DAY - offset_seconds) * MICROS_PER_SECOND + micros)
73+
let utc_micros = (days * SECONDS_PER_DAY - offset_seconds) * MICROS_PER_SECOND + micros;
74+
let day_micros = SECONDS_PER_DAY * MICROS_PER_SECOND;
75+
let first_micros = days_from_civil(MIN_YEAR, 1, 1) * day_micros;
76+
let end_micros = days_from_civil(MAX_YEAR + 1, 1, 1) * day_micros;
77+
if utc_micros < first_micros || utc_micros >= end_micros {
78+
return Err(Rejected);
79+
}
80+
Ok(utc_micros)
7481
}
7582

7683
fn date_days(date: &str) -> Result<i128, Rejected> {

‎crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ fn check_write_strategy<N: AuthoredNode>(
241241
return Err(Rejected);
242242
}
243243
reject_case_insensitive_duplicates(&excluded)?;
244-
let unique_columns = string_sequence(values.unique_key.as_ref())
244+
let unique_columns = string_sequence(values.unique_key.as_ref())?
245245
.iter()
246246
.map(|column| python_lower(column))
247247
.collect::<Result<HashSet<_>, _>>()?;
@@ -288,8 +288,8 @@ fn check_contract_columns<N: AuthoredNode>(
288288
.cursor
289289
.iter()
290290
.cloned()
291-
.chain(string_sequence(values.unique_key.as_ref()))
292-
.chain(string_sequence(values.merge_exclude_columns.as_ref()));
291+
.chain(string_sequence(values.unique_key.as_ref())?)
292+
.chain(string_sequence(values.merge_exclude_columns.as_ref())?);
293293
require_declared(named, &declared)
294294
}
295295

‎crates/sqlbuild-model-config/src/model_validation/_helpers/snapshot.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,9 @@ pub(crate) fn check_snapshot<N: AuthoredNode>(
9191
let checked_columns = if check_columns.is_some_and(is_wildcard_only) {
9292
Vec::new()
9393
} else {
94-
string_sequence(check_columns)
94+
string_sequence(check_columns)?
9595
};
96-
let named = string_sequence(unique_key)
96+
let named = string_sequence(unique_key)?
9797
.into_iter()
9898
.chain(updated_at)
9999
.chain(observed_at)

‎crates/sqlbuild-model-config/src/model_validation/constants.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,10 @@ pub const MAX_HOUR: i128 = 23;
205205
pub const MAX_MINUTE: i128 = 59;
206206
/// The length of a `HH:MM` UTC offset.
207207
pub const UTC_OFFSET_LENGTH: usize = 5;
208+
/// The first year Python's `datetime` represents.
209+
pub const MIN_YEAR: i128 = 1;
210+
/// The last year Python's `datetime` represents.
211+
pub const MAX_YEAR: i128 = 9999;
208212
/// The last month whose civil-day computation shifts the year back.
209213
pub const LAST_SHIFTED_MONTH: i128 = 2;
210214
/// The position of days among duration units.

‎crates/sqlbuild-model-config/src/model_validation/tests/test_accept_model_config.rs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,36 @@ fn given_effective_configs_when_validating_natively_then_only_python_valid_confi
6161
query_sql: "select 1",
6262
expected_accepted: false,
6363
},
64+
AcceptTestCase {
65+
description: "a start bound shifted before year 1 in UTC defers",
66+
config: incremental(vec![
67+
("cursor_start", Value::Str("0001-01-01T00:00:00+01:00")),
68+
("cursor_end", Value::Str("2024-01-01")),
69+
]),
70+
references: &["ref:orders"],
71+
query_sql: "select 1",
72+
expected_accepted: false,
73+
},
74+
AcceptTestCase {
75+
description: "an end bound shifted past year 9999 in UTC defers",
76+
config: incremental(vec![
77+
("cursor_start", Value::Str("2024-01-01")),
78+
("cursor_end", Value::Str("9999-12-31T23:59:00-01:00")),
79+
]),
80+
references: &["ref:orders"],
81+
query_sql: "select 1",
82+
expected_accepted: false,
83+
},
84+
AcceptTestCase {
85+
description: "bounds at the edges of years 1 and 9999 in UTC",
86+
config: incremental(vec![
87+
("cursor_start", Value::Str("0001-01-01T01:00:00+01:00")),
88+
("cursor_end", Value::Str("9999-12-31T22:59:00-01:00")),
89+
]),
90+
references: &["ref:orders"],
91+
query_sql: "select 1",
92+
expected_accepted: true,
93+
},
6494
AcceptTestCase {
6595
description: "an unusual ISO timestamp defers",
6696
config: incremental(vec![("cursor_start", Value::Str("2024-W01-1"))]),

‎src/sqlbuild/compiler/compile/_helpers/config/model_validation.py‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
)
4242
from sqlbuild.cursor_algebra.constants import GRAIN_BATCH_SIZE
4343
from sqlbuild.cursor_algebra.models import Duration
44+
from sqlbuild.errors.setting_help.main.model_header_help import model_header_help
4445
from sqlbuild.spec.contracts.constants import (
4546
CURSOR_POLICY_DISABLED,
4647
EFFECTIVE_BATCH_SIZE_TOKEN,
@@ -867,8 +868,12 @@ def _validate_incremental_cursor_rules(
867868
if (
868869
cursor_start is not None
869870
and cursor_end is not None
870-
and _cursor_contract_key(value=cursor_start, cursor_type=cursor_type)
871-
>= _cursor_contract_key(value=cursor_end, cursor_type=cursor_type)
871+
and _cursor_contract_key(
872+
value=cursor_start, cursor_type=cursor_type, key="cursor_start", model_name=model_name
873+
)
874+
>= _cursor_contract_key(
875+
value=cursor_end, cursor_type=cursor_type, key="cursor_end", model_name=model_name
876+
)
872877
):
873878
raise CompileInputError(
874879
f"model '{model_name}': cursor_start must be before exclusive cursor_end"
@@ -883,15 +888,26 @@ def _validate_incremental_cursor_rules(
883888
raise CompileInputError(f"model '{model_name}': append_cursor_inclusive requires cursor")
884889

885890

886-
def _cursor_contract_key(*, value: object, cursor_type: str | None) -> Decimal:
891+
def _cursor_contract_key(
892+
*, value: object, cursor_type: str | None, key: str, model_name: str
893+
) -> Decimal:
887894
if cursor_type == CursorType.INTEGER:
888895
return Decimal(str(value))
889896
parsed: datetime = datetime.fromisoformat(str(value))
890897
if parsed.tzinfo is None:
891-
parsed = parsed.replace(tzinfo=UTC)
892-
else:
893-
parsed = parsed.astimezone(UTC)
894-
return Decimal(str(parsed.timestamp()))
898+
return Decimal(str(parsed.replace(tzinfo=UTC).timestamp()))
899+
try:
900+
utc_parsed: datetime = parsed.astimezone(UTC)
901+
except OverflowError:
902+
raise CompileInputError(
903+
f"model '{model_name}': {key} value '{value}' falls outside years 1-9999 once "
904+
"converted to UTC",
905+
help=model_header_help(
906+
purpose=f"keep {key} within years 1-9999 in UTC",
907+
entry=f"{key} '{parsed.replace(tzinfo=UTC).isoformat()}'",
908+
),
909+
) from None
910+
return Decimal(str(utc_parsed.timestamp()))
895911

896912

897913
def _validate_cursor_safety_overrides(

‎tests/integration/src/sqlbuild/compiler/compile/native_model_config/_test_types.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,13 @@ class ModelConfigProjectTestCase:
3636
description: str
3737
engine: str
3838
expected_models: tuple[str, ...]
39+
40+
41+
@dataclass(frozen=True)
42+
class NativeRejectionParityTestCase:
43+
"""A config native validation must reject so Python reports its exact outcome."""
44+
45+
description: str
46+
values: dict[str, object]
47+
expected_native_accepted: bool
48+
expected_python_error: str

‎tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ def _reference(kind: str, name: str) -> CompileSqlReference:
193193
"2024-13-01",
194194
"20240101",
195195
"2024-W01-1",
196+
"0001-01-01T00:00:00+01:00",
196197
10,
197198
"15",
198199
"1e3",
@@ -201,8 +202,16 @@ def _reference(kind: str, name: str) -> CompileSqlReference:
201202
True,
202203
_ABSENT,
203204
),
204-
"cursor_end": ("2024-01-01", "2024-06-01 00:00", 5, 40, "30", "2025-01-01T00:00:00-05:00"),
205-
"unique_key": ("order_id", ["order_id", "customer_id"], [], (), 5, _ABSENT),
205+
"cursor_end": (
206+
"2024-01-01",
207+
"2024-06-01 00:00",
208+
5,
209+
40,
210+
"30",
211+
"2025-01-01T00:00:00-05:00",
212+
"9999-12-31T23:59:00-01:00",
213+
),
214+
"unique_key": ("order_id", ["order_id", "customer_id"], ["\udcff"], [], (), 5, _ABSENT),
206215
"merge_exclude_columns": (["amount"], ["ORDER_ID"], ["a", "A"], [""], "amount", _ABSENT),
207216
"incremental_mode": ("full", "microbatch", "batch", _ABSENT),
208217
"microbatch_strategy": ("rolling_window", "watermark", "hourly", _ABSENT),
@@ -247,7 +256,7 @@ def _reference(kind: str, name: str) -> CompileSqlReference:
247256
"initial_valid_from": ("updated_at", "observed_at", "execution_time", "now", _ABSENT),
248257
"snapshot_full_refresh": ("deny", "allow", "always", _ABSENT),
249258
"snapshot_schema_change": ("deny", "append_new_columns", "drop", _ABSENT),
250-
"check_columns": (["*"], ["*", "status"], ["status"], [], ("*",), _ABSENT),
259+
"check_columns": (["*"], ["*", "status"], ["status"], ["\udcff"], [], ("*",), _ABSENT),
251260
"invalidate_hard_deletes": (True, False, "yes", _ABSENT),
252261
"valid_from_column": ("valid_from", "VALID_TO", _ABSENT),
253262
"valid_to_column": ("valid_to", _ABSENT),
@@ -342,6 +351,35 @@ def model_validation_parity(*, requests: list[ModelValidationRequest]) -> ModelV
342351
)
343352

344353

354+
def validation_outcomes(*, values: dict[str, object]) -> tuple[bool, str]:
355+
"""Return whether native validation accepts `values` and the Python validators' error."""
356+
357+
request: ModelValidationRequest = ModelValidationRequest(
358+
model_file=_model_file(relative_path="marts/orders_daily.sql", header_values={}),
359+
config=CompileModelConfig(values=values),
360+
references=(_ORDERS,),
361+
declared_columns=None,
362+
query_sql="SELECT 1",
363+
)
364+
session: NativeModelConfigSession = _session(
365+
project_config=ProjectConfig(name="orders", adapter="duckdb"),
366+
target_config=None,
367+
effective_target_name=None,
368+
)
369+
return (
370+
native_model_validators_accept(session=session, request=request),
371+
_python_error(request),
372+
)
373+
374+
375+
def _python_error(request: ModelValidationRequest) -> str:
376+
try:
377+
run_python_model_validators(context=_VALIDATOR_CONTEXT, request=request)
378+
except Exception as error: # noqa: BLE001 - the exact Python outcome, whatever it is
379+
return f"{type(error).__name__}: {error}"
380+
return "accepted"
381+
382+
345383
def _validation_request(*, rng: random.Random) -> ModelValidationRequest:
346384
profile, references = rng.choice(_PROFILES)
347385
values: dict[str, object] = dict(profile)

‎tests/integration/src/sqlbuild/compiler/compile/native_model_config/test_native_model_config_session.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from tests.integration.src.sqlbuild.compiler.compile.native_model_config._test_types import (
1010
ModelConfigBuildParityTestCase,
1111
ModelValidationParityTestCase,
12+
NativeRejectionParityTestCase,
1213
)
1314
from tests.integration.src.sqlbuild.compiler.compile.native_model_config.helpers import (
1415
BUILD_ENVIRONMENT,
@@ -18,6 +19,7 @@
1819
generated_validation_requests,
1920
model_config_build_parity,
2021
model_validation_parity,
22+
validation_outcomes,
2123
)
2224

2325

@@ -90,5 +92,87 @@ def test_given_generated_layers_when_building_config_then_native_matches_python_
9092
) == ([], True, True, True), (parity.built, parity.deferred, parity.python_raised)
9193

9294

95+
@pytest.mark.parametrize(
96+
"test_case",
97+
[
98+
NativeRejectionParityTestCase(
99+
description="a start bound shifted before year 1 in UTC",
100+
values={
101+
"materialized": "incremental",
102+
"incremental_strategy": "append",
103+
"cursor": "updated_at",
104+
"cursor_type": "timestamp",
105+
"cursor_grain": "day",
106+
"cursor_start": "0001-01-01T00:00:00+01:00",
107+
"cursor_end": "2024-01-01",
108+
},
109+
expected_native_accepted=False,
110+
expected_python_error=(
111+
"CompileInputError: model 'orders_daily': cursor_start value "
112+
"'0001-01-01T00:00:00+01:00' falls outside years 1-9999 once converted to UTC"
113+
),
114+
),
115+
NativeRejectionParityTestCase(
116+
description="an end bound shifted past year 9999 in UTC",
117+
values={
118+
"materialized": "incremental",
119+
"incremental_strategy": "append",
120+
"cursor": "updated_at",
121+
"cursor_type": "timestamp",
122+
"cursor_grain": "day",
123+
"cursor_start": "2024-01-01",
124+
"cursor_end": "9999-12-31T23:59:00-01:00",
125+
},
126+
expected_native_accepted=False,
127+
expected_python_error=(
128+
"CompileInputError: model 'orders_daily': cursor_end value "
129+
"'9999-12-31T23:59:00-01:00' falls outside years 1-9999 once converted to UTC"
130+
),
131+
),
132+
NativeRejectionParityTestCase(
133+
description="an unreadable unique key under an enforced contract",
134+
values={
135+
"materialized": "incremental",
136+
"incremental_strategy": "merge",
137+
"unique_key": ["\udcff"],
138+
"contract": "enforced",
139+
"columns": {"order_id": None},
140+
},
141+
expected_native_accepted=False,
142+
expected_python_error=(
143+
"CompileInputError: model 'orders_daily': unique_key references column "
144+
"'\udcff' not declared in enforced contract"
145+
),
146+
),
147+
NativeRejectionParityTestCase(
148+
description="an unreadable snapshot check column under an enforced contract",
149+
values={
150+
"materialized": "snapshot",
151+
"unique_key": "order_id",
152+
"snapshot_strategy": "check",
153+
"check_columns": ["\udcff"],
154+
"contract": "enforced",
155+
"columns": {"order_id": None},
156+
},
157+
expected_native_accepted=False,
158+
expected_python_error=(
159+
"CompileInputError: model 'orders_daily': check_columns references column "
160+
"'\udcff' not declared in enforced contract"
161+
),
162+
),
163+
],
164+
ids=lambda case: case.description,
165+
)
166+
def test_given_python_rejected_edge_configs_when_validating_then_native_defers_to_python(
167+
test_case: NativeRejectionParityTestCase,
168+
) -> None:
169+
accepted, python_error = validation_outcomes(values=test_case.values)
170+
171+
assert (accepted, python_error) == (
172+
test_case.expected_native_accepted,
173+
test_case.expected_python_error,
174+
)
175+
176+
93177
if __name__ == "__main__":
94178
pytest.main([__file__, "-vv"])

0 commit comments

Comments
 (0)