Skip to content

Commit ec968d0

Browse files
authored
refactor(native): read planner models and project collections through native views (#1200)
* refactor(native): read compiled models through one counted native accessor `model_reads::native_model` returns the native view behind a model object and counts every other object, a Python-built record that is read by attribute instead, in the `compiled_view_fallbacks` stage counter. Every model read helper goes through it. An integration test shows compile and plan of a native project never fall back, and a unit test shows Python-built models are still read, and counted. * refactor(native): read resolution and strategy models through native views Planner resolution, strategy, scope and change detection read a model's schema entry, config, references, inferred columns, relative path, name and dependency rows, and iterate the project's models, sources, seeds, functions and audits, through the native view helpers instead of attribute reads. Reads that serve every resource kind keep the shared attribute dispatch. * refactor(native): read warehouse and migration models through native views Planner warehouse and migration code read a model's name, destination, key, schema entry, config, references and dependency keys, and iterate the project's models, seeds, functions and sources, through the native view helpers. Reads that serve every resource kind keep the shared attribute dispatch. * refactor(native): read remaining planner models and project collections natively Plan entries, phases, identity metadata, scenarios, selectors and the audit, seed, test, display and clone command plans read a model's name, destination, key, schema entry, config, references and enum columns, and iterate the project's models, sources, seeds, functions, audits and SQL tests, through the native view helpers. A named collection lookup serves the callers that pick the collection by name.
1 parent e1a2850 commit ec968d0

58 files changed

Lines changed: 563 additions & 198 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns";
3434
pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns";
3535
/// Config values converted into typed nodes for a key a reader asked for.
3636
pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads";
37+
/// Reads of a model or project that is not a native compiled view, read by attribute instead.
38+
pub(crate) const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks";
3739

3840
/// Compile stage runs since the last reset, by stage.
3941
#[pyclass]

‎crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ pub(crate) mod inputs_view;
88
pub(crate) mod model_input_view;
99
pub(crate) mod model_reads;
1010
pub(crate) mod project_host;
11+
pub(crate) mod project_reads;
1112
pub(crate) mod project_view;
1213
pub(crate) mod records;
1314
pub(crate) mod registration;

‎crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs‎

Lines changed: 114 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
//! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields.
22
3-
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
3+
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
44

5+
use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage};
56
use crate::bindings::_helpers::compiled_views::compiled_model_view::{
67
DeclaredRow, LineageInput, NativeCompiledModel,
78
};
@@ -27,10 +28,22 @@ const TRANSFORM_VALUES: [&str; 6] = [
2728
];
2829
const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"];
2930

31+
/// The native view behind a model object; any other object is a Python-built record, counted.
32+
#[inline]
33+
pub(crate) fn native_model<'a>(
34+
model: &'a Bound<'_, PyAny>,
35+
) -> PyResult<Option<&'a NativeCompiledModel>> {
36+
if let Ok(view) = model.cast::<NativeCompiledModel>() {
37+
return Ok(Some(view.get()));
38+
}
39+
count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?;
40+
Ok(None)
41+
}
42+
3043
/// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred.
3144
pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<InferredRow>>> {
32-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
33-
return view.get().inferred_rows(model.py());
45+
if let Some(view) = native_model(model)? {
46+
return view.inferred_rows(model.py());
3447
}
3548
let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?;
3649
if rows.is_none() {
@@ -56,24 +69,23 @@ fn column_names(rows: Vec<InferredRow>) -> Vec<String> {
5669

5770
/// Each declared column of the attached schema as `(name, type)`, `None` without a schema.
5871
pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<DeclaredRow>>> {
59-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
60-
return view.get().schema_column_rows(model.py());
72+
if let Some(view) = native_model(model)? {
73+
return view.schema_column_rows(model.py());
6174
}
6275
native_field(model, "schema_column_rows")?.extract()
6376
}
6477

6578
/// Whether the attached schema declares dynamic column families.
6679
pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult<bool> {
67-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
68-
return view.get().dynamic_columns_declared(model.py());
80+
if let Some(view) = native_model(model)? {
81+
return view.dynamic_columns_declared(model.py());
6982
}
7083
native_field(model, "has_dynamic_columns")?.is_truthy()
7184
}
7285

7386
/// `(has star, star resolved)` of the model's analysed query.
7487
pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> {
75-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
76-
let view: &NativeCompiledModel = view.get();
88+
if let Some(view) = native_model(model)? {
7789
return Ok((
7890
view.fast_lineage_has_star(model.py())?,
7991
view.fast_lineage_star_resolved(model.py())?,
@@ -87,8 +99,8 @@ pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> {
8799

88100
/// Output columns whose types semantic recovery forgot, in any order.
89101
pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult<Vec<String>> {
90-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
91-
return Ok(view.get().unchecked_columns().to_vec());
102+
if let Some(view) = native_model(model)? {
103+
return Ok(view.unchecked_columns().to_vec());
92104
}
93105
native_field(model, "unchecked_output_columns")?
94106
.try_iter()?
@@ -98,8 +110,8 @@ pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult<Vec<String
98110

99111
/// Each lineage output with the `(resource, column)` pairs it reads; empty without lineage.
100112
pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult<Vec<LineageInput>> {
101-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
102-
return Ok(view.get().lineage_inputs(model.py())?.unwrap_or_default());
113+
if let Some(view) = native_model(model)? {
114+
return Ok(view.lineage_inputs(model.py())?.unwrap_or_default());
103115
}
104116
let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?;
105117
if !columns.is_truthy()? {
@@ -123,25 +135,25 @@ pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult<Vec<LineageIn
123135

124136
/// Whether the native session validated the model's bindings.
125137
pub(crate) fn binding_validated(model: &Bound<'_, PyAny>) -> PyResult<bool> {
126-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
127-
return view.get().binding_validated(model.py());
138+
if let Some(view) = native_model(model)? {
139+
return view.binding_validated(model.py());
128140
}
129141
native_field(model, "binding_validated")?.is_truthy()
130142
}
131143

132144
/// Whether the model's dynamic pivot proof proved its output columns.
133145
pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult<bool> {
134-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
135-
return view.get().output_proven(model.py());
146+
if let Some(view) = native_model(model)? {
147+
return view.output_proven(model.py());
136148
}
137149
let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?;
138150
Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?)
139151
}
140152

141153
/// Each lineage output with its transform kind and confidence values, `None` without lineage.
142154
pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<LineageFactRow>>> {
143-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
144-
let Some(rows) = view.get().lineage_rows(model.py())? else {
155+
if let Some(view) = native_model(model)? {
156+
let Some(rows) = view.lineage_rows(model.py())? else {
145157
return Ok(None);
146158
};
147159
let mut facts: Vec<LineageFactRow> = Vec::with_capacity(rows.len());
@@ -190,8 +202,8 @@ fn coded(values: &[&str], code: u8) -> PyResult<String> {
190202

191203
/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`.
192204
pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<String>>> {
193-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
194-
return view.get().proven_inputs(model.py());
205+
if let Some(view) = native_model(model)? {
206+
return view.proven_inputs(model.py());
195207
}
196208
let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?;
197209
if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? {
@@ -202,41 +214,112 @@ pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult<Option<Vec<Str
202214

203215
/// The model's name.
204216
pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult<String> {
205-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
206-
return view.get().name(model.py());
217+
if let Some(view) = native_model(model)? {
218+
return view.name(model.py());
207219
}
208220
native_field(model, "name")?.extract()
209221
}
210222

211223
/// The model's compiled query SQL.
212224
pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
213-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
214-
return Ok(view.get().query_sql(model.py())?.into_any());
225+
if let Some(view) = native_model(model)? {
226+
return Ok(view.query_sql(model.py())?.into_any());
215227
}
216228
native_field(model, "query_sql")
217229
}
218230

219231
/// The model's `CompiledObjectKey`.
220232
pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
221-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
222-
return Ok(view.get().key(model.py())?.into_bound(model.py()));
233+
if let Some(view) = native_model(model)? {
234+
return Ok(view.key(model.py())?.into_bound(model.py()));
223235
}
224236
native_field(model, "key")
225237
}
226238

227239
/// The model's destination `CompiledRelationLocation`.
228240
pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
229-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
230-
return Ok(view.get().destination(model.py())?.into_bound(model.py()));
241+
if let Some(view) = native_model(model)? {
242+
return Ok(view.destination(model.py())?.into_bound(model.py()));
231243
}
232244
native_field(model, "destination")
233245
}
234246

235247
/// The model's name as a Python string.
236248
pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
237-
if let Ok(view) = model.downcast::<NativeCompiledModel>() {
238-
let name: String = view.get().name(model.py())?;
249+
if let Some(view) = native_model(model)? {
250+
let name: String = view.name(model.py())?;
239251
return Ok(pyo3::types::PyString::new(model.py(), &name).into_any());
240252
}
241253
native_field(model, "name")
242254
}
255+
256+
/// The model's attached `SchemaModelEntry`, or `None`.
257+
pub(crate) fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
258+
let py: Python<'py> = model.py();
259+
if let Some(view) = native_model(model)? {
260+
return Ok(view.schema_entry(py)?.into_bound(py));
261+
}
262+
native_field(model, "schema_entry")
263+
}
264+
265+
/// The model's `CompileModelConfig`.
266+
pub(crate) fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
267+
let py: Python<'py> = model.py();
268+
if let Some(view) = native_model(model)? {
269+
return Ok(view.config(py)?.into_bound(py));
270+
}
271+
native_field(model, "config")
272+
}
273+
274+
/// The model's `CompileSqlReference` tuple.
275+
pub(crate) fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
276+
let py: Python<'py> = model.py();
277+
if let Some(view) = native_model(model)? {
278+
return Ok(view.references(py)?.into_bound(py).into_any());
279+
}
280+
native_field(model, "references")
281+
}
282+
283+
/// The model file's relative path.
284+
pub(crate) fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
285+
let py: Python<'py> = model.py();
286+
if let Some(view) = native_model(model)? {
287+
return Ok(view.relative_path(py)?.into_bound(py));
288+
}
289+
native_field(model, "relative_path")
290+
}
291+
292+
/// The model's `InferredColumn` tuple, or `None`.
293+
pub(crate) fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
294+
let py: Python<'py> = model.py();
295+
if let Some(view) = native_model(model)? {
296+
return Ok(view.inferred_columns(py)?.into_bound(py));
297+
}
298+
native_field(model, "inferred_columns")
299+
}
300+
301+
/// `(resource type, name)` of each dependency.
302+
pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult<Vec<(String, String)>> {
303+
if let Some(view) = native_model(model)? {
304+
return view.dep_rows(model.py());
305+
}
306+
native_field(model, "dep_rows")?.extract()
307+
}
308+
309+
/// The model's dependency `CompiledObjectKey` tuple.
310+
pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
311+
let py: Python<'py> = model.py();
312+
if let Some(view) = native_model(model)? {
313+
return Ok(view.deps(py)?.into_bound(py).into_any());
314+
}
315+
native_field(model, "deps")
316+
}
317+
318+
/// The model's enum columns by name.
319+
pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
320+
let py: Python<'py> = model.py();
321+
if let Some(view) = native_model(model)? {
322+
return Ok(view.enum_columns(py)?.into_bound(py).into_any());
323+
}
324+
native_field(model, "enum_columns")
325+
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
//! A compiled project's resource collections, read from its native view.
2+
3+
use pyo3::prelude::{Bound, PyAny, PyResult, Python};
4+
5+
use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage};
6+
use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView;
7+
use crate::bindings::_helpers::records::fields::field as native_field;
8+
9+
const MODELS_FIELD: &str = "models";
10+
const SOURCES_FIELD: &str = "sources";
11+
const SEEDS_FIELD: &str = "seeds";
12+
const FUNCTIONS_FIELD: &str = "functions";
13+
const AUDITS_FIELD: &str = "audits";
14+
15+
/// The native view behind a project object; any other object is a Python-built record, counted.
16+
#[inline]
17+
fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult<Option<&'a NativeProjectView>> {
18+
if let Ok(view) = project.cast::<NativeProjectView>() {
19+
return Ok(Some(view.get()));
20+
}
21+
count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?;
22+
Ok(None)
23+
}
24+
25+
/// The project's compiled models, in compile order.
26+
pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
27+
let py: Python<'py> = project.py();
28+
if let Some(view) = native_project(project)? {
29+
return Ok(view.models(py).into_bound(py).into_any());
30+
}
31+
native_field(project, MODELS_FIELD)
32+
}
33+
34+
/// The project's compiled sources.
35+
pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
36+
let py: Python<'py> = project.py();
37+
if let Some(view) = native_project(project)? {
38+
return Ok(view.sources(py).into_bound(py));
39+
}
40+
native_field(project, SOURCES_FIELD)
41+
}
42+
43+
/// The project's compiled functions.
44+
pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
45+
let py: Python<'py> = project.py();
46+
if let Some(view) = native_project(project)? {
47+
return Ok(view.functions(py).into_bound(py));
48+
}
49+
native_field(project, FUNCTIONS_FIELD)
50+
}
51+
52+
/// The project's compiled seeds.
53+
pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
54+
let py: Python<'py> = project.py();
55+
if let Some(view) = native_project(project)? {
56+
return Ok(view.seeds(py).into_bound(py));
57+
}
58+
native_field(project, SEEDS_FIELD)
59+
}
60+
61+
/// The project's compiled audits.
62+
pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
63+
let py: Python<'py> = project.py();
64+
if let Some(view) = native_project(project)? {
65+
return Ok(view.audits(py).into_bound(py));
66+
}
67+
native_field(project, AUDITS_FIELD)
68+
}
69+
70+
/// The project's resource collection named `field`; any other field is read by attribute.
71+
pub(crate) fn collection<'py>(
72+
project: &Bound<'py, PyAny>,
73+
field: &str,
74+
) -> PyResult<Bound<'py, PyAny>> {
75+
match field {
76+
MODELS_FIELD => models(project),
77+
SOURCES_FIELD => sources(project),
78+
SEEDS_FIELD => seeds(project),
79+
FUNCTIONS_FIELD => functions(project),
80+
AUDITS_FIELD => audits(project),
81+
_ => native_field(project, field),
82+
}
83+
}
84+
85+
/// The project's compiled SQL tests.
86+
pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
87+
let py: Python<'py> = project.py();
88+
if let Some(view) = native_project(project)? {
89+
return Ok(view.sql_tests(py).into_bound(py));
90+
}
91+
native_field(project, "sql_tests")
92+
}

‎crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost};
44

55
use crate::bindings::_helpers::compiled_views::model_reads;
6+
use crate::bindings::_helpers::compiled_views::project_reads;
67
use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families;
78
use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values};
89
use crate::bindings::_helpers::planner_changes::results::native_enum;
@@ -107,7 +108,7 @@ pub(crate) fn detect_changes<'py>(
107108
let query_change_tracking: bool =
108109
native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?;
109110
let dialect: Option<String> = native_field(snapshot, "column_dialect")?.extract()?;
110-
for function in native_field(project, "functions")?.try_iter()? {
111+
for function in project_reads::functions(project)?.try_iter()? {
111112
let function: Bound<'py, PyAny> = function?;
112113
let name: Bound<'py, PyAny> = native_field(&function, "name")?;
113114
let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?;
@@ -248,7 +249,7 @@ pub(crate) fn scope_model_changes<'py>(
248249
}
249250
};
250251
let function_local_hashes: Bound<'py, PyAny> = function_local_hashes(
251-
&native_field(project, "functions")?,
252+
&project_reads::functions(project)?,
252253
native_field(project, "sql_analysis_dialect")?
253254
.extract::<Option<String>>()?
254255
.as_deref(),
@@ -530,7 +531,7 @@ fn model_schema_findings<'py>(
530531
for column in &validity {
531532
warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name));
532533
}
533-
let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?;
534+
let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?;
534535
let mut yml: Vec<(String, String)> = Vec::new();
535536
let mut type_enforcement: bool = false;
536537
let mut dynamic: Option<Bound<'py, PyAny>> = None;

0 commit comments

Comments
 (0)