Skip to content

Commit bf6845f

Browse files
committed
Merge remote-tracking branch 'origin/main' into feat/native-model-analysis-enrichment
# Conflicts: # tests/e2e/scripts/compiler_differential/test_compiler_differential.py
2 parents a86b008 + 9a8e410 commit bf6845f

92 files changed

Lines changed: 5242 additions & 126 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
//! Python's `collect_model_column_contract_diagnostics`, in its exact emission order.
2+
3+
use std::collections::{HashMap, HashSet};
4+
5+
use crate::contracts::_helpers::dynamic::dynamic_column_diagnostics;
6+
use crate::contracts::_helpers::type_comparison::{TypeComparison, types_equal};
7+
use crate::contracts::constants::{
8+
CONTRACT_ENFORCED, CONTRACT_NONE, DYNAMIC_OUTPUT_NOT_PROVEN_CODE, EXTRA_COLUMN_CODE,
9+
EXTRA_COLUMN_HELP, EXTRA_COLUMN_NAMED_SCHEMA_HELP, MISSING_COLUMN_CODE,
10+
MISSING_DECLARATIONS_CODE, MISSING_DECLARATIONS_HELP, MODEL_COLUMNS_DECLARATION,
11+
NAMED_SCHEMA_DECLARATION, NULLABILITY_HELP, NULLABILITY_MISMATCH_CODE, NULLABLE_OUTPUT_MESSAGE,
12+
TYPE_MISMATCH_CODE, TYPE_MISMATCH_HELP, UNKNOWN_TYPE_CODE, UNKNOWN_TYPE_HELP,
13+
UNPROVEN_TYPE_OUTPUT_MESSAGE,
14+
};
15+
use crate::contracts::models::{
16+
ContractDeferral, ContractDiagnostic, ContractLocation, ContractModel, ContractSchema,
17+
ContractSeverity, DeclaredColumn, InferredOutputColumn, RelatedOutput,
18+
};
19+
20+
/// Python's `_requires_contract_evaluation`.
21+
pub(crate) fn requires_contract_evaluation(model: &ContractModel, implicit: bool) -> bool {
22+
declared_shape_validation_is_active(model, implicit)
23+
|| model
24+
.schema
25+
.as_ref()
26+
.is_some_and(|schema| schema.type_enforcement)
27+
}
28+
29+
/// Python's `_declared_shape_validation_is_active`.
30+
pub(crate) fn declared_shape_validation_is_active(model: &ContractModel, implicit: bool) -> bool {
31+
match model.contract.as_deref() {
32+
Some(CONTRACT_ENFORCED) => true,
33+
Some(CONTRACT_NONE) => false,
34+
_ => {
35+
implicit
36+
&& model
37+
.schema
38+
.as_ref()
39+
.is_some_and(|schema| !schema.columns.is_empty())
40+
}
41+
}
42+
}
43+
44+
/// One model's contract diagnostics, or the reason Python must evaluate it.
45+
pub(crate) fn model_contract_diagnostics(
46+
model: &ContractModel,
47+
validate_declared_shape: bool,
48+
dialect: &str,
49+
) -> Result<Vec<ContractDiagnostic>, ContractDeferral> {
50+
let contract_enforced: bool = model.contract.as_deref() == Some(CONTRACT_ENFORCED);
51+
let Some(schema) = model
52+
.schema
53+
.as_ref()
54+
.filter(|schema| !schema.columns.is_empty() || !schema.dynamic_columns.is_empty())
55+
else {
56+
return Ok(if contract_enforced {
57+
vec![missing_declarations(model)]
58+
} else {
59+
Vec::new()
60+
});
61+
};
62+
let mut diagnostics: Vec<ContractDiagnostic> = dynamic_column_diagnostics(model, dialect)?;
63+
let dynamic_output_unproven: bool = diagnostics
64+
.iter()
65+
.any(|diagnostic| diagnostic.code == DYNAMIC_OUTPUT_NOT_PROVEN_CODE);
66+
let Some(inferred_columns) = model.inferred_columns.as_ref() else {
67+
return Ok(diagnostics);
68+
};
69+
let inferred_by_name: HashMap<&str, &InferredOutputColumn> = inferred_columns
70+
.iter()
71+
.map(|column| (column.name.as_str(), column))
72+
.collect();
73+
if contract_enforced {
74+
diagnostics.extend(extra_columns(model, schema, inferred_columns));
75+
}
76+
let unchecked: HashSet<&str> = model
77+
.unchecked_output_columns
78+
.iter()
79+
.map(String::as_str)
80+
.collect();
81+
for (index, declared) in schema.columns.iter().enumerate() {
82+
let Some(inferred) = inferred_by_name.get(declared.name.as_str()) else {
83+
if validate_declared_shape && !model.fast_lineage_has_star {
84+
diagnostics.push(missing_column(model, index, declared, contract_enforced));
85+
}
86+
continue;
87+
};
88+
if validate_declared_shape {
89+
diagnostics.extend(nullability(index, declared, inferred));
90+
}
91+
let Some(declared_type) = declared.declared_type.as_deref() else {
92+
continue;
93+
};
94+
if !validate_declared_shape && !schema.type_enforcement {
95+
continue;
96+
}
97+
if dynamic_output_unproven && inferred.inferred_type.is_none() {
98+
continue;
99+
}
100+
let checked = TypedColumn {
101+
index,
102+
declared,
103+
declared_type,
104+
inferred,
105+
};
106+
let check = TypeCheck {
107+
type_enforcement: schema.type_enforcement,
108+
contract_enforced,
109+
unchecked: &unchecked,
110+
dialect,
111+
};
112+
diagnostics.extend(column_type(&checked, &check)?);
113+
}
114+
Ok(diagnostics)
115+
}
116+
117+
struct TypeCheck<'a> {
118+
type_enforcement: bool,
119+
contract_enforced: bool,
120+
unchecked: &'a HashSet<&'a str>,
121+
dialect: &'a str,
122+
}
123+
124+
struct TypedColumn<'a> {
125+
index: usize,
126+
declared: &'a DeclaredColumn,
127+
declared_type: &'a str,
128+
inferred: &'a InferredOutputColumn,
129+
}
130+
131+
fn missing_declarations(model: &ContractModel) -> ContractDiagnostic {
132+
ContractDiagnostic {
133+
code: MISSING_DECLARATIONS_CODE,
134+
severity: ContractSeverity::Error,
135+
message: format!(
136+
"model '{}' has contract enforced but declares no columns",
137+
model.name
138+
),
139+
column_name: None,
140+
location: ContractLocation::None,
141+
related_output: None,
142+
help: MISSING_DECLARATIONS_HELP.to_owned(),
143+
}
144+
}
145+
146+
fn extra_columns(
147+
model: &ContractModel,
148+
schema: &ContractSchema,
149+
inferred_columns: &[InferredOutputColumn],
150+
) -> Vec<ContractDiagnostic> {
151+
let declared_names: HashSet<&str> = schema
152+
.columns
153+
.iter()
154+
.map(|column| column.name.as_str())
155+
.collect();
156+
let help: &str = if schema.named_schema {
157+
EXTRA_COLUMN_NAMED_SCHEMA_HELP
158+
} else {
159+
EXTRA_COLUMN_HELP
160+
};
161+
inferred_columns
162+
.iter()
163+
.filter(|column| !declared_names.contains(column.name.as_str()))
164+
.map(|column| ContractDiagnostic {
165+
code: EXTRA_COLUMN_CODE,
166+
severity: ContractSeverity::Error,
167+
message: format!(
168+
"column '{}' is not declared in enforced contract for model '{}'",
169+
column.name, model.name
170+
),
171+
column_name: Some(column.name.clone()),
172+
location: ContractLocation::Output(column.name.clone()),
173+
related_output: None,
174+
help: help.to_owned(),
175+
})
176+
.collect()
177+
}
178+
179+
fn nullability(
180+
index: usize,
181+
declared: &DeclaredColumn,
182+
inferred: &InferredOutputColumn,
183+
) -> Option<ContractDiagnostic> {
184+
if !declared.not_null || !inferred.nullable {
185+
return None;
186+
}
187+
Some(ContractDiagnostic {
188+
code: NULLABILITY_MISMATCH_CODE,
189+
severity: ContractSeverity::Error,
190+
message: format!(
191+
"column '{}' is declared non-null but may be nullable",
192+
declared.name
193+
),
194+
column_name: Some(declared.name.clone()),
195+
location: ContractLocation::Declared(index),
196+
related_output: Some(RelatedOutput {
197+
column_name: declared.name.clone(),
198+
message: NULLABLE_OUTPUT_MESSAGE.to_owned(),
199+
}),
200+
help: NULLABILITY_HELP.to_owned(),
201+
})
202+
}
203+
204+
fn missing_column(
205+
model: &ContractModel,
206+
index: usize,
207+
column: &DeclaredColumn,
208+
contract_enforced: bool,
209+
) -> ContractDiagnostic {
210+
let contract_label: &str = if contract_enforced {
211+
"enforced contract column"
212+
} else {
213+
"declared column"
214+
};
215+
let declaration_label: &str = if column.declared_in_named_schema {
216+
NAMED_SCHEMA_DECLARATION
217+
} else {
218+
MODEL_COLUMNS_DECLARATION
219+
};
220+
let name: &str = &column.name;
221+
let help: String = if contract_enforced {
222+
format!(
223+
"add {name} to the SELECT list or correct {declaration_label}; validation is active \
224+
because this model declares contract enforced"
225+
)
226+
} else {
227+
format!(
228+
"add {name} to the SELECT list or correct/remove {declaration_label}; \
229+
{declaration_label} is validated using static SQL analysis because \
230+
settings.column_contract_mode is \"implicit\" (the default). If this project \
231+
intentionally uses columns only for metadata and audits, set [settings] \
232+
column_contract_mode = \"explicit\"; models with contract enforced remain validated"
233+
)
234+
};
235+
ContractDiagnostic {
236+
code: MISSING_COLUMN_CODE,
237+
severity: ContractSeverity::Error,
238+
message: format!(
239+
"{contract_label} '{name}' was not found in statically inferred output for model '{}'",
240+
model.name
241+
),
242+
column_name: Some(name.to_owned()),
243+
location: ContractLocation::Declared(index),
244+
related_output: None,
245+
help,
246+
}
247+
}
248+
249+
fn column_type(
250+
column: &TypedColumn<'_>,
251+
check: &TypeCheck<'_>,
252+
) -> Result<Option<ContractDiagnostic>, ContractDeferral> {
253+
let name: &str = &column.declared.name;
254+
let declared_type: &str = column.declared_type;
255+
let Some(inferred_type) = column.inferred.inferred_type.as_deref() else {
256+
if !check.type_enforcement || check.unchecked.contains(column.inferred.name.as_str()) {
257+
return Ok(None);
258+
}
259+
return Ok(Some(ContractDiagnostic {
260+
code: UNKNOWN_TYPE_CODE,
261+
severity: ContractSeverity::Warning,
262+
message: format!(
263+
"column '{name}' type could not be proven against declared {declared_type}"
264+
),
265+
column_name: Some(name.to_owned()),
266+
location: ContractLocation::Declared(column.index),
267+
related_output: Some(RelatedOutput {
268+
column_name: name.to_owned(),
269+
message: UNPROVEN_TYPE_OUTPUT_MESSAGE.to_owned(),
270+
}),
271+
help: UNKNOWN_TYPE_HELP.to_owned(),
272+
}));
273+
};
274+
if types_equal(declared_type, inferred_type, check.dialect)? == TypeComparison::Equal {
275+
return Ok(None);
276+
}
277+
Ok(Some(ContractDiagnostic {
278+
code: TYPE_MISMATCH_CODE,
279+
severity: if check.type_enforcement || check.contract_enforced {
280+
ContractSeverity::Error
281+
} else {
282+
ContractSeverity::Warning
283+
},
284+
message: format!(
285+
"column '{name}' inferred as {inferred_type} but declared type is {declared_type}"
286+
),
287+
column_name: Some(name.to_owned()),
288+
location: ContractLocation::Declared(column.index),
289+
related_output: Some(RelatedOutput {
290+
column_name: name.to_owned(),
291+
message: format!("inferred {inferred_type}"),
292+
}),
293+
help: TYPE_MISMATCH_HELP.to_owned(),
294+
}))
295+
}

0 commit comments

Comments
 (0)