diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 88d543f7fa..12cac440d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -34,6 +34,8 @@ pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; /// Config values converted into typed nodes for a key a reader asked for. pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads"; +/// Reads of a model or project that is not a native compiled view, read by attribute instead. +pub(crate) const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks"; /// Compile stage runs since the last reset, by stage. #[pyclass] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs index a8c2429d4e..ecdb130732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod inputs_view; pub(crate) mod model_input_view; pub(crate) mod model_reads; pub(crate) mod project_host; +pub(crate) mod project_reads; pub(crate) mod project_view; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index 752cddb036..fb24ddc151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -1,7 +1,8 @@ //! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ DeclaredRow, LineageInput, NativeCompiledModel, }; @@ -27,10 +28,22 @@ const TRANSFORM_VALUES: [&str; 6] = [ ]; const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; +/// The native view behind a model object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_model<'a>( + model: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = model.cast::() { + return Ok(Some(view.get())); + } + count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} + /// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().inferred_rows(model.py()); + if let Some(view) = native_model(model)? { + return view.inferred_rows(model.py()); } let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; if rows.is_none() { @@ -56,24 +69,23 @@ fn column_names(rows: Vec) -> Vec { /// Each declared column of the attached schema as `(name, type)`, `None` without a schema. pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().schema_column_rows(model.py()); + if let Some(view) = native_model(model)? { + return view.schema_column_rows(model.py()); } native_field(model, "schema_column_rows")?.extract() } /// Whether the attached schema declares dynamic column families. pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().dynamic_columns_declared(model.py()); + if let Some(view) = native_model(model)? { + return view.dynamic_columns_declared(model.py()); } native_field(model, "has_dynamic_columns")?.is_truthy() } /// `(has star, star resolved)` of the model's analysed query. pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { - if let Ok(view) = model.downcast::() { - let view: &NativeCompiledModel = view.get(); + if let Some(view) = native_model(model)? { return Ok(( view.fast_lineage_has_star(model.py())?, view.fast_lineage_star_resolved(model.py())?, @@ -87,8 +99,8 @@ pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { /// Output columns whose types semantic recovery forgot, in any order. pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().unchecked_columns().to_vec()); + if let Some(view) = native_model(model)? { + return Ok(view.unchecked_columns().to_vec()); } native_field(model, "unchecked_output_columns")? .try_iter()? @@ -98,8 +110,8 @@ pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().lineage_inputs(model.py())?.unwrap_or_default()); + if let Some(view) = native_model(model)? { + return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); } let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; if !columns.is_truthy()? { @@ -123,16 +135,16 @@ pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().binding_validated(model.py()); + if let Some(view) = native_model(model)? { + return view.binding_validated(model.py()); } native_field(model, "binding_validated")?.is_truthy() } /// Whether the model's dynamic pivot proof proved its output columns. pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().output_proven(model.py()); + if let Some(view) = native_model(model)? { + return view.output_proven(model.py()); } let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) @@ -140,8 +152,8 @@ pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { /// Each lineage output with its transform kind and confidence values, `None` without lineage. pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - let Some(rows) = view.get().lineage_rows(model.py())? else { + if let Some(view) = native_model(model)? { + let Some(rows) = view.lineage_rows(model.py())? else { return Ok(None); }; let mut facts: Vec = Vec::with_capacity(rows.len()); @@ -190,8 +202,8 @@ fn coded(values: &[&str], code: u8) -> PyResult { /// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().proven_inputs(model.py()); + if let Some(view) = native_model(model)? { + return view.proven_inputs(model.py()); } let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { @@ -202,41 +214,112 @@ pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().name(model.py()); + if let Some(view) = native_model(model)? { + return view.name(model.py()); } native_field(model, "name")?.extract() } /// The model's compiled query SQL. pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().query_sql(model.py())?.into_any()); + if let Some(view) = native_model(model)? { + return Ok(view.query_sql(model.py())?.into_any()); } native_field(model, "query_sql") } /// The model's `CompiledObjectKey`. pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().key(model.py())?.into_bound(model.py())); + if let Some(view) = native_model(model)? { + return Ok(view.key(model.py())?.into_bound(model.py())); } native_field(model, "key") } /// The model's destination `CompiledRelationLocation`. pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().destination(model.py())?.into_bound(model.py())); + if let Some(view) = native_model(model)? { + return Ok(view.destination(model.py())?.into_bound(model.py())); } native_field(model, "destination") } /// The model's name as a Python string. pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - let name: String = view.get().name(model.py())?; + if let Some(view) = native_model(model)? { + let name: String = view.name(model.py())?; return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); } native_field(model, "name") } + +/// The model's attached `SchemaModelEntry`, or `None`. +pub(crate) fn schema_entry<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.schema_entry(py)?.into_bound(py)); + } + native_field(model, "schema_entry") +} + +/// The model's `CompileModelConfig`. +pub(crate) fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.config(py)?.into_bound(py)); + } + native_field(model, "config") +} + +/// The model's `CompileSqlReference` tuple. +pub(crate) fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.references(py)?.into_bound(py).into_any()); + } + native_field(model, "references") +} + +/// The model file's relative path. +pub(crate) fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.relative_path(py)?.into_bound(py)); + } + native_field(model, "relative_path") +} + +/// The model's `InferredColumn` tuple, or `None`. +pub(crate) fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.inferred_columns(py)?.into_bound(py)); + } + native_field(model, "inferred_columns") +} + +/// `(resource type, name)` of each dependency. +pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return view.dep_rows(model.py()); + } + native_field(model, "dep_rows")?.extract() +} + +/// The model's dependency `CompiledObjectKey` tuple. +pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.deps(py)?.into_bound(py).into_any()); + } + native_field(model, "deps") +} + +/// The model's enum columns by name. +pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.enum_columns(py)?.into_bound(py).into_any()); + } + native_field(model, "enum_columns") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs new file mode 100644 index 0000000000..a189750f87 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs @@ -0,0 +1,92 @@ +//! A compiled project's resource collections, read from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::records::fields::field as native_field; + +const MODELS_FIELD: &str = "models"; +const SOURCES_FIELD: &str = "sources"; +const SEEDS_FIELD: &str = "seeds"; +const FUNCTIONS_FIELD: &str = "functions"; +const AUDITS_FIELD: &str = "audits"; + +/// The native view behind a project object; any other object is a Python-built record, counted. +#[inline] +fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult> { + if let Ok(view) = project.cast::() { + return Ok(Some(view.get())); + } + count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} + +/// The project's compiled models, in compile order. +pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.models(py).into_bound(py).into_any()); + } + native_field(project, MODELS_FIELD) +} + +/// The project's compiled sources. +pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sources(py).into_bound(py)); + } + native_field(project, SOURCES_FIELD) +} + +/// The project's compiled functions. +pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.functions(py).into_bound(py)); + } + native_field(project, FUNCTIONS_FIELD) +} + +/// The project's compiled seeds. +pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.seeds(py).into_bound(py)); + } + native_field(project, SEEDS_FIELD) +} + +/// The project's compiled audits. +pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.audits(py).into_bound(py)); + } + native_field(project, AUDITS_FIELD) +} + +/// The project's resource collection named `field`; any other field is read by attribute. +pub(crate) fn collection<'py>( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + match field { + MODELS_FIELD => models(project), + SOURCES_FIELD => sources(project), + SEEDS_FIELD => seeds(project), + FUNCTIONS_FIELD => functions(project), + AUDITS_FIELD => audits(project), + _ => native_field(project, field), + } +} + +/// The project's compiled SQL tests. +pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sql_tests(py).into_bound(py)); + } + native_field(project, "sql_tests") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index bac3ace9cd..3aeb87e01c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; @@ -107,7 +108,7 @@ pub(crate) fn detect_changes<'py>( let query_change_tracking: bool = native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = native_field(&function, "name")?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; @@ -248,7 +249,7 @@ pub(crate) fn scope_model_changes<'py>( } }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &native_field(project, "functions")?, + &project_reads::functions(project)?, native_field(project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -530,7 +531,7 @@ fn model_schema_findings<'py>( for column in &validity { warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); } - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index df127086a8..d54dd25725 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -240,7 +240,7 @@ fn changed_referencing_query( continue; } let mut references_models: bool = false; - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { if native_field(&reference?, "ref_kind")?.eq(&ref_kind)? { references_models = true; break; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 6896a71a80..5150cc3d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; @@ -128,13 +129,13 @@ fn resolved_source_entries<'py>( let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let read = PySet::empty(py)?; let mut readers: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if recreated_view(&model, &selected)? { readers.push(model); } } - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if selected.contains(native_field(&function, "key")?)? { readers.push(function); @@ -164,14 +165,11 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = object( - py, - build_model_locations(&native_field(project, "models")?)?, - )?; - let seed_locations = object(py, build_seed_locations(&native_field(project, "seeds")?)?)?; + let model_locations = object(py, build_model_locations(&project_reads::models(project)?)?)?; + let seed_locations = object(py, build_seed_locations(&project_reads::seeds(project)?)?)?; let function_locations = object( py, - build_function_locations(&native_field(project, "functions")?)?, + build_function_locations(&project_reads::functions(project)?)?, )?; let source_map = PyDict::new(py); for source in native_field(source_project, "sources")?.try_iter()? { @@ -239,7 +237,7 @@ fn clone_model_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; @@ -247,7 +245,7 @@ fn clone_model_entries<'py>( let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; functions.set_item(native_field(&function, "name")?, function)?; } @@ -308,7 +306,7 @@ fn clone_source_entries<'py>( ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let entry = native_field(&source, "source_entry")?; if !selected.contains(native_field(&source, "key")?)? @@ -366,7 +364,7 @@ fn clone_function_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if !selected.contains(native_field(&function, "key")?)? { continue; @@ -452,7 +450,7 @@ fn seed_entries<'py>( wanted: &Bound<'py, PyAny>, ) -> PyResult> { let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if wanted.contains(native_field(&seed, field)?)? { entries.push(seed_entry(&seed)?); @@ -467,7 +465,7 @@ fn origin_model_entries<'py>( ) -> PyResult> { let py = project.py(); let mut entries: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; @@ -494,7 +492,7 @@ fn origin_source_entries<'py>( let py = project.py(); let mut keys: Vec> = Vec::new(); let mut order: Vec> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if selected_names.contains(native_field(&source, "name")?)? { keys.push(native_field(&source, "key")?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 6de0009d69..759b7fa9a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ @@ -159,7 +160,7 @@ pub(crate) fn build_audit_command_plan<'py>( ) -> PyResult> { let py = project.py(); let materializations = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; materializations.set_item( model_reads::name_object(&model)?, @@ -200,7 +201,7 @@ fn called_function_keys<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let functions: Bound<'py, PyDict> = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; functions.set_item(native_field(&function, "key")?, function)?; } @@ -254,7 +255,7 @@ pub(crate) fn build_test_command_plan<'py>( )?; let (entries, warnings) = (selected.entries, selected.warnings); let mut case_names: BTreeSet = BTreeSet::new(); - for test in native_field(&project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(&project)?.try_iter()? { let test = test?; let case_name = native_field(&test, "case_name")?; if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { @@ -271,7 +272,7 @@ pub(crate) fn build_test_command_plan<'py>( } let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); - for function in native_field(&project, "functions")?.try_iter()? { + for function in project_reads::functions(&project)?.try_iter()? { let function = function?; if !function_keys.contains(native_field(&function, "key")?)? { continue; @@ -331,7 +332,7 @@ pub(crate) fn build_seed_command_plan<'py>( }; let selected_keys = native_field(scope, "selected_keys")?; let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys.contains(native_field(&seed, "key")?)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index caf8713de1..1d67e3d651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,6 +1,7 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, @@ -138,7 +139,7 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .collect::>>()?, )?; let mut entries: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if selected.contains(model_reads::name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index ae9d789bc0..ddba156a81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; @@ -426,7 +427,7 @@ pub(crate) fn plan_entries<'py>( runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); - for function in native_field(&project, "functions")?.try_iter()? { + for function in project_reads::functions(&project)?.try_iter()? { let function = function?; functions_by_name.set_item(native_field(&function, "name")?, function)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index afbf586a68..8aebad4969 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ @@ -104,7 +105,7 @@ fn seed_entries<'py>( let metadata_jsons = native_field(extras, "seed_metadata_jsons")?; let plan_reasons = native_field(extras, "seed_plan_reasons")?; let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys.contains(native_field(&seed, "key")?)? { continue; @@ -404,7 +405,7 @@ fn function_entries<'py>( let functions = native_field(changes, "functions")?; let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let mut entries: Vec> = Vec::new(); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if !selected_keys.contains(native_field(&function, "key")?)? { continue; @@ -464,7 +465,7 @@ pub(super) fn selected_test_entries<'py>( ) -> PyResult> { let py = project.py(); let mut selected: Vec> = Vec::new(); - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test = test?; if !test_selected(&test, selected_keys, selection)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index b04b65bc6c..1e4797cbac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; @@ -31,7 +32,7 @@ fn selected_audit_relation_keys<'py>( ) -> PyResult> { let py = project.py(); let keys = PySet::empty(py)?; - for audit in native_field(project, "audits")?.try_iter()? { + for audit in project_reads::audits(project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, selected_keys)? { for dependency in native_field(&audit, "scope_deps")?.try_iter()? { @@ -99,12 +100,12 @@ pub(crate) fn planner_relations_context<'py>( let selected_keys = native_field(&scope, "selected_keys")?; let mut model_locations = object( py, - build_model_locations(&native_field(&project, "models")?)?, + build_model_locations(&project_reads::models(&project)?)?, )?; - let mut seed_locations = object(py, build_seed_locations(&native_field(&project, "seeds")?)?)?; + let mut seed_locations = object(py, build_seed_locations(&project_reads::seeds(&project)?)?)?; let function_locations = object( py, - build_function_locations(&native_field(&project, "functions")?)?, + build_function_locations(&project_reads::functions(&project)?)?, )?; let deferred_locations = native_field(&deferral, "deferred_locations")?; if !deferred_locations.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 3130f2b735..40ac2bf9cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -20,6 +20,7 @@ use sqlbuild_analysis::planner_identity::main::topological_order::topological_or use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_identity::hashing::graph_key; @@ -89,8 +90,14 @@ pub(crate) fn build_execution_upstream_deps<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let upstream: Bound<'py, PyDict> = PyDict::new(py); - for collection in ["models", "sources", "seeds", "functions"] { - for resource in native_field(project, collection)?.try_iter()? { + let collections: [Bound<'py, PyAny>; 4] = [ + project_reads::models(project)?, + project_reads::sources(project)?, + project_reads::seeds(project)?, + project_reads::functions(project)?, + ]; + for collection in collections { + for resource in collection.try_iter()? { let resource: Bound<'py, PyAny> = resource?; upstream.set_item( native_field(&resource, "key")?, @@ -103,7 +110,7 @@ pub(crate) fn build_execution_upstream_deps<'py>( )?; } } - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; let test_key: Bound<'py, PyAny> = native_field(&test, "key")?; let scope_deps: Bound<'py, PyAny> = native_field(&test, "scope_deps")?; @@ -172,7 +179,7 @@ pub(crate) fn build_execution_edge_origins<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let origins: Bound<'py, PyDict> = PyDict::new(py); - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; let test_name: String = native_field(&test, "name")?.extract()?; for target_key in native_field(&test, "scope_deps")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 0f59644b3a..0e56c7adad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -180,7 +180,7 @@ pub(crate) fn direct_model_version_identities<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name), IdentityNode { - upstream_keys: graph_keys(&native_field(&model, "deps")?)?, + upstream_keys: graph_keys(&model_reads::deps(&model)?)?, local_hash: Some(local_hash), }, ); @@ -291,7 +291,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { - upstream_keys: graph_keys(&native_field(&model, "deps")?)?, + upstream_keys: graph_keys(&model_reads::deps(&model)?)?, local_hash: mapping_get(&local_hashes, &name)?.extract()?, }, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 10e6618d2b..3be7c2a31b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -90,7 +90,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let py: Python<'py> = model.py(); let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in native_field(model, "deps")?.try_iter()? { + for upstream_key in model_reads::deps(model)?.try_iter()? { let upstream_key: Bound<'py, PyAny> = upstream_key?; let kind: String = native_field(&upstream_key, "resource_type")? .str()? @@ -105,7 +105,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( } } } - let config_values: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")?; + let config_values: Bound<'py, PyAny> = native_field(&model_reads::config(model)?, "values")?; let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( @@ -170,7 +170,7 @@ fn contract_output_signature<'py>( config_values: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = model.py(); - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; if schema_entry.is_none() { return Ok(None); } @@ -184,7 +184,7 @@ fn contract_output_signature<'py>( if !enforced && native_field(&schema_entry, "model_schema")?.is_none() { return Ok(None); } - let enum_columns: Bound<'py, PyAny> = native_field(model, "enum_columns")?; + let enum_columns: Bound<'py, PyAny> = model_reads::enum_columns(model)?; let rendered: Bound<'py, PyList> = PyList::empty(py); for column in columns.try_iter()? { let column: Bound<'py, PyAny> = column?; @@ -247,7 +247,7 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'_, PyAny> = model_reads::schema_entry(model)?; let payload: Bound<'_, PyDict> = PyDict::new(py); payload.set_item( "contract", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index beb4f4ff94..6cca98b4c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -45,7 +45,7 @@ pub(super) fn candidates<'py>( let warehouse = native_field(&snapshot, "existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let declared = PyDict::new(py); - let schema = native_field(&model, "schema_entry")?; + let schema = model_reads::schema_entry(&model)?; if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { let column = column?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index 2cab5a269b..77549f4fac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,6 +1,7 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::matching::{ match_migration_fingerprints, migration_destination_fingerprints, }; @@ -56,7 +57,7 @@ pub(super) fn discover_model_migrations<'py>( let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); for item in &models { - names.set_item(native_field(item, "name")?, item)?; + names.set_item(model_reads::name_object(item)?, item)?; } let event_renames = discovery_events::renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { @@ -74,7 +75,7 @@ pub(super) fn discover_model_migrations<'py>( } let unbuilt = PyDict::new(py); for item in &models { - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; if !declared_names.contains(&name)? && call_with( &native_field(&state, "named_fingerprint")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 9bec76268f..966ce3f899 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; @@ -110,11 +111,11 @@ pub(super) fn resumed<'py>( let target = native_field(&native_field(runtime, "project")?, "effective_target_name")?; let events = native_field(state, "events")?; for item in models { - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { continue; } - let destination = native_field(item, "destination")?; + let destination = model_reads::destination(item)?; let event = newest(&events, &destination)?; if event.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 861a44763d..38899cfdf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -2,6 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; @@ -19,16 +21,13 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - let name: String = native_field(&item, "name")?.extract()?; + let name: String = model_reads::name_object(&item)?.extract()?; let mut upstream: Vec = Vec::new(); - for dependency in native_field(&item, "deps")?.try_iter()? { + for dependency in model_reads::deps(&item)?.try_iter()? { let dependency = dependency?; if native_field(&dependency, "resource_type")?.eq("model")? { upstream.push(native_field(&dependency, "name")?.extract()?); @@ -63,8 +62,8 @@ pub(super) fn triggered( let relevant = relevant.0; let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?; for item in models { - let name = native_field(item, "name")?; - if relevant.contains(native_field(item, "key")?)? + let name = model_reads::name_object(item)?; + if relevant.contains(model_reads::model_key(item)?)? && !fingerprints.contains(&name)? && !declared.contains(&name)? { @@ -83,7 +82,7 @@ pub(super) fn metadata<'py>( let functions = object( py, build_function_local_hashes( - &native_field(&project, "functions")?, + &project_reads::functions(&project)?, native_field(&project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -102,7 +101,7 @@ pub(super) fn metadata<'py>( let result = PyDict::new(py); for item in models { result.set_item( - native_field(item, "name")?, + model_reads::name_object(item)?, object( py, build_model_version_identity_metadata_json( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index a8bde508ea..b0dd9cf867 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -1,6 +1,7 @@ //! Live candidate inspection and automatic request materialization. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ @@ -114,10 +115,10 @@ pub(super) fn automatic<'py>( let py = runtime.py(); let mut requests: Vec> = Vec::new(); for item in models { - if !native_field(scope, "selected_keys")?.contains(native_field(item, "key")?)? { + if !native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { continue; } - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; let event = matches.1.get_item(&name)?; let candidate = mapping_get(matches.0, &name)?; let origin = if let Some(event) = event { @@ -162,8 +163,8 @@ pub(super) fn warnings<'py>( let py = scope.py(); let selected = PySet::empty(py)?; for item in models { - if native_field(scope, "selected_keys")?.contains(native_field(item, "key")?)? { - selected.add(native_field(item, "name")?)?; + if native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { + selected.add(model_reads::name_object(item)?)?; } } let ambiguous: BTreeMap = ambiguous.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 344d3ab855..bfda1b78be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; @@ -73,7 +74,7 @@ pub(super) fn decide<'py>( let py = runtime.py(); let request = decision_request.request; let item = request.model.clone(); - let destination = native_field(&item, "destination")?; + let destination = model_reads::destination(&item)?; let origin = decision_request.origin.clone(); let snapshot = decision_request.snapshot.clone(); let state = decision_request.state.clone(); @@ -99,7 +100,7 @@ pub(super) fn decide<'py>( py, RecordId::ModelMigrationPlanEntry, &[ - ("model_name", native_field(&item, "name")?), + ("model_name", model_reads::name_object(&item)?), ("discovery", request.discovery.clone()), ( "decision", @@ -160,7 +161,7 @@ pub(super) fn decide<'py>( } let inherited = if recorded && !applies(&item, &newest, &snapshot)? { native_field(&native_field(&snapshot, "fingerprints")?, "models")? - .call_method1("get", (native_field(&item, "name")?,))? + .call_method1("get", (model_reads::name_object(&item)?,))? } else { empty_version(&item, &fingerprint)? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 453a186d4c..2a90e97ead 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,6 +2,8 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{ config_node_of, config_value, model_values, }; @@ -97,11 +99,8 @@ fn discover<'py>( let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in native_field( - &runtime_field(&runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? { if let Some(request) = declared(&item?)? { declared_requests.push(request); @@ -117,7 +116,7 @@ fn discover<'py>( for request in &declared_requests { let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; declarations.push(ModelMigrationDeclaration { - model_name: native_field(&request.model, "name")?, + model_name: model_reads::name_object(&request.model)?, origin_location, origin_model, }); @@ -183,7 +182,7 @@ fn finish<'py>( let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); - let name = native_field(&item, "name")?; + let name = model_reads::name_object(&item)?; let decided = super::model_entries::decide(&MigrationDecisionRequest { runtime: runtime.clone(), request, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 33e58dcdfe..be08c9830c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; @@ -113,8 +114,7 @@ impl<'py> OldNamePlanner<'py> { &self, item: &Bound<'py, PyAny>, ) -> PyResult>> { - let Some(history) = self.newest(&relation(&native_field(item, "destination")?)?, None)? - else { + let Some(history) = self.newest(&relation(&model_reads::destination(item)?)?, None)? else { return Ok(None); }; if !native_field(&native_field(&history, "move")?, "destination_model")? @@ -324,7 +324,7 @@ impl<'py> OldNamePlanner<'py> { py, RecordId::OldNameViewPlanEntry, &[ - ("model_name", native_field(item, "name")?), + ("model_name", model_reads::name_object(item)?), ( "origin", location(&self.runtime, &native_field(history, "old")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 840ef0b8ab..1aaa832716 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; @@ -42,7 +43,7 @@ pub(super) fn retention<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { old_name_retention( - &native_field(&native_field(item, "config")?, "values")?, + &native_field(&model_reads::config(item)?, "values")?, project, ) } @@ -70,7 +71,7 @@ pub(super) fn recorded_view<'py>( let created = native_field(history, "created")?; let mut reused = py.None().into_bound(py); for item in models { - if relation(&native_field(item, "destination")?)? + if relation(&model_reads::destination(item)?)? .call_method1("matches", (native_field(history, "old")?,))? .is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 16654086ab..a7eb581cc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -4,6 +4,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; @@ -24,7 +26,7 @@ fn schemas<'py>( let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { - let schema = native_field(&native_field(item, "destination")?, "schema")?; + let schema = native_field(&model_reads::destination(item)?, "schema")?; if !schema.is_none() { needed.add(schema)?; } @@ -43,7 +45,7 @@ fn schemas<'py>( if !include_project { for item in selected { let existing = native_field(&snapshot, "existing_relations")? - .call_method1("get", (native_field(item, "name")?,))?; + .call_method1("get", (model_reads::name_object(item)?,))?; if !existing.is_none() && host::entry(py, AdapterHost::NormalizeRelationType)? .call1((native_field(&existing, "relation_type")?,))? @@ -75,12 +77,12 @@ pub(crate) fn plan_old_name_views<'py>( let runtime = phase.runtime.clone(); let scope = phase.scope.clone(); let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let models: Vec> = native_field(&project, "models")? + let models: Vec> = project_reads::models(&project)? .try_iter()? .collect::>()?; let by_name = PyDict::new(py); for item in &models { - by_name.set_item(native_field(item, "name")?, item)?; + by_name.set_item(model_reads::name_object(item)?, item)?; } let mut selected: Vec> = Vec::new(); for key in native_field(&scope, "execution_order")?.try_iter()? { @@ -136,7 +138,7 @@ pub(crate) fn plan_old_name_views<'py>( } } for item in &selected { - if !covered.contains(native_field(item, "name")?)? + if !covered.contains(model_reads::name_object(item)?)? && let Some(planned) = planner.pending_for(item)? { entries.push(planned); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 09fabbd24d..516a9a1277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -22,7 +22,7 @@ pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { - let schema = native_field(model, "schema_entry")?; + let schema = model_reads::schema_entry(model)?; let mut columns: Vec<(String, Option)> = Vec::new(); if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 8ae3f96ed6..4639fdef9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -1,6 +1,8 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -31,14 +33,11 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - if native_field(&item, "name")?.eq(name)? { + if model_reads::name_object(&item)?.eq(name)? { return Ok((native_field(&item, "destination")?, name.clone())); } } @@ -73,8 +72,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; - for item in native_field(&project, "models")?.try_iter()? { - let database = native_field(&native_field(&item?, "destination")?, "database")?; + for item in project_reads::models(&project)?.try_iter()? { + let database = native_field(&model_reads::destination(&item?)?, "database")?; if !database.is_none() { return Ok(database); } @@ -85,13 +84,10 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { - let schema = native_field(&native_field(&item?, "destination")?, "schema")?; + let schema = native_field(&model_reads::destination(&item?)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -145,7 +141,7 @@ pub(super) fn resolve_origin<'py>( _ => super::column_objects::text(py, ""), }; let item = request.model.clone(); - let destination = native_field(&item, "destination")?; + let destination = model_reads::destination(&item)?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { parts.push( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 7f5241fcb3..fb58fa50d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,6 +1,7 @@ //! Selected-scope buildability validation before change planning. use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_identity::graph::build_execution_edge_origins; use crate::bindings::_helpers::planner_identity::graph::check_buildability; use crate::bindings::_helpers::planner_identity::graph::missing_upstream_message; @@ -28,7 +29,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( let project = runtime_field(runtime, |runtime| &runtime.project)?; let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let mut external_seeds: Vec<_> = Vec::new(); - for seed in native_field(&project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(&project)?.try_iter()? { let seed = seed?; if native_field(&seed, "external")?.is_truthy()? { external_seeds.push(native_field(&seed, "key")?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index fc61483b4a..cef30a198f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,6 +1,7 @@ //! Per-model full-refresh resolution and full-rebuild refusals. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; @@ -39,7 +40,7 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( cli_full_refresh: bool, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { names.push(model_reads::name_object(&model)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index a869e16d2a..2125419581 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,6 +1,7 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; @@ -49,10 +50,10 @@ fn version_identities<'py>( ) -> PyResult> { identity_work(project.py())?.identity_builds += 1; direct_model_version_identities( - (&native_field(project, "functions")?, &scope), + (&project_reads::functions(project)?, &scope), native_field(project, "sql_analysis_dialect")?.extract()?, ( - &native_field(project, "seeds")?, + &project_reads::seeds(project)?, &project.py().None().into_bound(project.py()), &native_field(project, "hook_functions")?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 4376b8d325..acac698320 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::HostId; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; @@ -150,7 +151,7 @@ pub(crate) fn by_name<'py>( field: &str, ) -> PyResult> { let nodes = pyo3::types::PyDict::new(project.py()); - for node in native_field(project, field)?.try_iter()? { + for node in project_reads::collection(project, field)?.try_iter()? { let node = node?; pyo3::types::PyDictMethods::set_item(&nodes, native_field(&node, "name")?, node)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 5e7c1c134f..82db7755f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; @@ -175,11 +176,8 @@ pub(crate) fn add_storage_policies<'py>( let mut entries = output.field("model_entries")?; if native_field(policies, "record_migration_fingerprints")?.is_truthy()? { let models = PyDict::new(py); - for model in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for model in project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)? + .try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index f48921bebd..680e3d1de2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -4,6 +4,7 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, typed_values}; use crate::bindings::_helpers::native_host::host; @@ -62,11 +63,8 @@ fn selected_models<'py>( .into_iter() .collect(); let mut models: Vec<_> = Vec::new(); - for model in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for model in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let model = model?; if names.contains(&model_reads::model_name(&model)?) { @@ -94,7 +92,7 @@ pub(crate) fn plan_table_types<'py>( let mut declared: Vec<_> = Vec::new(); for model in &models { if native_field( - &native_field(&native_field(model, "config")?, "table_type")?, + &native_field(&model_reads::config(model)?, "table_type")?, "declared", )? .is_truthy()? @@ -595,7 +593,7 @@ pub(crate) fn plan_retention<'py>( let py = runtime.py(); let mut models: Vec<_> = Vec::new(); for model in selected_models(runtime, scope)? { - let retention = native_field(&native_field(&model, "config")?, "time_travel_retention")?; + let retention = native_field(&model_reads::config(&model)?, "time_travel_retention")?; if !native_field(&retention, "unmanaged")?.is_truthy()? && !native_field(&retention, "desired_days")?.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 39b573ed81..15e418f408 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; @@ -138,7 +139,7 @@ fn namespace<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let models = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index e506f21877..561b359710 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; @@ -23,7 +24,7 @@ pub(super) fn seed_entries<'py>( relations: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let seeds = by_attribute(&native_field(project, "seeds")?, "name")?; + let seeds = by_attribute(&project_reads::seeds(project)?, "name")?; let fixtures = native_field(graph, "seed_fixture_names")?; let mut result: Vec> = Vec::new(); for name in native_field(graph, "seed_names")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index e209fbbc4c..7328fe60e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; @@ -89,9 +90,9 @@ pub(crate) fn build_scenario_execution_plan<'py>( } else { analysis.is_truthy()? }; - let models = by_attribute(&native_field(&project, "models")?, "name")?; - let functions = by_attribute(&native_field(&project, "functions")?, "key")?; - let locations = build_function_locations(&native_field(&project, "functions")?)?.into_any(); + let models = by_attribute(&project_reads::models(&project)?, "name")?; + let functions = by_attribute(&project_reads::functions(&project)?, "key")?; + let locations = build_function_locations(&project_reads::functions(&project)?)?.into_any(); let upstream = build_execution_upstream_deps(&project)?.into_any(); let ordered = object(py, topologically_order_keys(&upstream, None)?)?; let mut model_keys: Vec> = Vec::new(); @@ -267,7 +268,7 @@ fn model_context<'py>( ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), models_by_name: models.clone().into_any().unbind(), - functions_by_name: by_attribute(&native_field(project, "functions")?, "name")? + functions_by_name: by_attribute(&project_reads::functions(project)?, "name")? .into_any() .unbind(), seed_locations: native_field(&context.relations, "seed_locations")?.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 525142eac8..29039b9dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,6 +1,7 @@ //! Convert project views into pure scenario dependency facts. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::values::string; @@ -33,7 +34,7 @@ pub(crate) fn plan_scenario_graph<'py>( dbt_refs: native_field(scenario, "dbt_ref_fixture_names")?.extract()?, ..ScenarioGraphInput::default() }; - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let deps = native_field(&model, "deps")? .try_iter()? @@ -43,12 +44,12 @@ pub(crate) fn plan_scenario_graph<'py>( .models .insert(model_reads::name_object(&model)?.extract()?, deps); } - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { input .project_sources .insert(native_field(&source?, "name")?.extract()?); } - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { input .project_seeds .insert(native_field(&seed?, "name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index 30ae9b2874..3ea5d202b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -1,6 +1,8 @@ //! Planner scope assembly using native graph and selector operations. use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::graph::views::lineage_graph_names; @@ -33,7 +35,7 @@ pub(crate) fn managed_source_upstream_keys<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let mut managed: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if !native_field(&native_field(&source, "source_entry")?, "loader")?.is_none() { managed.insert(native_field(&source, "name")?.extract()?); @@ -119,9 +121,9 @@ pub(crate) fn build_planner_scope<'py>( let origins = build_execution_edge_origins(&project)?.into_any(); let order = topologically_order_keys(&upstream, Some(origins))?.into_any(); let models = PyDict::new(py); - for model in native_field(&project, "models")?.try_iter()? { + for model in project_reads::models(&project)?.try_iter()? { let model = model?; - models.set_item(native_field(&model, "name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let mut reads = args.value("python_read_source_names")?; if reads.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs index eb354739b7..3b789fa489 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs @@ -2,6 +2,7 @@ use sqlbuild_host::entries::models::ExecutorHost; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::algorithms::closure; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; @@ -30,7 +31,7 @@ pub(crate) fn build_source_load_map<'py>( }) .transpose()?; let result = PyDict::new(project.py()); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if let Some(keys) = &relevant && !keys.contains(native_field(&source, "key")?)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs index 0058a12dec..8102e74d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -1,5 +1,6 @@ //! Python function identity and compiled graph conversion. +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -67,7 +68,7 @@ pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; graph.sources.push(( native_field(&entry, "name")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs index 34c0701681..78848545cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::planner_selection::models::{ }; use crate::bindings::_helpers::boundary::python_calls::imported; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_selection::carrier::Carrier; use crate::bindings::_helpers::records::construct::new_record_with; @@ -88,7 +89,7 @@ pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( term: &Bound<'py, PyString>, ) -> PyResult> { let py: Python<'py> = project.py(); - let models: Vec = native_field(project, "models")? + let models: Vec = project_reads::models(project)? .try_iter()? .map(|model| selectable_model(&model?)) .collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 189e4be386..f4dcdde7be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::auditing::render::render_audit_sql; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::views::project_lineage_views; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::audits::{ @@ -229,7 +230,7 @@ pub(crate) fn build_selected_audit_entries<'py>( let project = request.get("project")?; let selected_keys = native_field(&request.get("scope")?, "selected_keys")?; let mut selected: Vec> = Vec::new(); - for audit in native_field(&project, "audits")?.try_iter()? { + for audit in project_reads::audits(&project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, &selected_keys)? { selected.push(audit); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 7f9fb73a19..dc3f0b3088 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -13,6 +13,7 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_strategy::constants::{ MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, }; @@ -53,7 +54,7 @@ pub(crate) fn planning_context( Some(models) => models, None => { let models = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, &model)?; } @@ -67,11 +68,16 @@ pub(crate) fn planning_context( model_fixture_relation(&model)?, ); } - for (collection, entry, kind, source) in [ - ("sources", "source_entry", SOURCE_REFERENCE_KIND, true), - ("seeds", "schema_entry", SEED_REFERENCE_KIND, false), + for (entry, kind, source) in [ + ("source_entry", SOURCE_REFERENCE_KIND, true), + ("schema_entry", SEED_REFERENCE_KIND, false), ] { - for resource in native_field(project, collection)?.try_iter()? { + let resources: Bound<'_, PyAny> = if source { + project_reads::sources(project)? + } else { + project_reads::seeds(project)? + }; + for resource in resources.try_iter()? { let resource = resource?; let entry = native_field(&resource, entry)?; let columns: Vec = columns(&native_field(&entry, "columns")?, false)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index edd894a333..c488412173 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -5,6 +5,7 @@ use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; @@ -18,7 +19,7 @@ const CONTRACT_KEY: &str = "contract"; /// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { let inferred: Option> = inferred_rows(model)?; - let schema = native_field(model, "schema_entry")?; + let schema = model_reads::schema_entry(model)?; let declared: Vec = if schema.is_none() { Vec::new() } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 20f5bdbd1f..aacaa6ffab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -62,7 +62,7 @@ pub(crate) fn required_fixture_columns( continue; }; let mut references: Vec = Vec::new(); - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { let reference = reference?; references.push(FixtureReadReference { kind: native_field(&reference, "ref_kind")?.extract()?, @@ -132,7 +132,7 @@ fn model_stars( return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for key in native_field(model, "dep_rows")?.extract::>()? { + for key in model_reads::dep_rows(model)? { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index e25f1f94c8..cf58cc1fd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -86,8 +86,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { fn producer_facts(&self, name: &str) -> PyResult { let producer: Bound<'py, PyAny> = self.models_by_name.get_item(name)?; - let values: Bound<'py, PyAny> = - native_field(&native_field(&producer, "config")?, "values")?; + let values: Bound<'py, PyAny> = native_field(&model_reads::config(&producer)?, "values")?; let capped: bool = match microbatch_limit_config(&values)?.1 { Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), None => false, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 260780475d..009cd9e0b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -195,7 +195,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; if !entry.is_none() { let enforcement: Bound<'py, PyAny> = native_field(&entry, "type_enforcement")?; if !enforcement.is_none() { @@ -310,7 +310,7 @@ fn logical_ddl<'py>( /// The output column names SQL analysis inferred for the model, when it inferred them. fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; + let inferred: Bound<'py, PyAny> = model_reads::inferred_columns(model)?; if inferred.is_none() { return Ok(none(py)); } @@ -577,7 +577,7 @@ fn contract_fields<'py>( mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?, )?; entry.set_item("contract_columns", schema_columns(model, false)?)?; - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; entry.set_item( "contract_dynamic_columns", if schema_entry.is_none() { @@ -653,7 +653,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( .unwrap_or_else(|| PyTuple::empty(py)); let table_type: Bound<'py, PyAny> = native_field( &native_field( - &native_field(&native_field(model, "config")?, "table_type")?, + &native_field(&model_reads::config(model)?, "table_type")?, "value", )?, "value", @@ -666,7 +666,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let entry: Bound<'py, PyDict> = PyDict::new(py); entry.set_item("key", model_reads::model_key(model)?)?; entry.set_item("name", &name)?; - entry.set_item("relative_path", native_field(model, "relative_path")?)?; + entry.set_item("relative_path", model_reads::relative_path(model)?)?; entry.set_item("destination", model_reads::destination(model)?)?; entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 974ff73a14..ff17cb0f39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::cursor::intervals::partition; use crate::bindings::_helpers::cursor::scalars; use crate::bindings::_helpers::cursor::scalars::compare_values; @@ -194,7 +195,7 @@ fn runtime_producer_names( return Ok(names); } let mut loaded: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if !native_field(&native_field(&source, "source_entry")?, "loader")?.is_none() { loaded.insert(native_field(&source, "name")?.extract()?); @@ -341,7 +342,7 @@ fn collect_models<'py>( }; let runtime_names = runtime_producer_names(project, &producers)?; let mut models: Vec<_> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let name_value = model_reads::name_object(&model)?; let model_name: String = name_value.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index a63794be55..e64a922859 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; @@ -192,7 +193,7 @@ pub(crate) fn semantic_source_columns<'py>( return Ok(columns.clone()); } let mut required: BTreeSet = BTreeSet::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let analysis_off: bool = matches!( *config_node_of(&model_values(&model)?, SQL_ANALYSIS_KEY)?, @@ -201,7 +202,7 @@ pub(crate) fn semantic_source_columns<'py>( if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis_off { continue; } - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { let reference = reference?; if native_field(&reference, "ref_kind")?.eq("source")? { required.insert(native_field(&reference, "ref_name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index eb6a4f00d0..d208b15f31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; @@ -75,8 +76,13 @@ pub(crate) fn destination_nodes<'py>( project: &Bound<'py, PyAny>, ) -> PyResult>> { let mut nodes: Vec<_> = Vec::new(); - for field in ["models", "seeds", "functions"] { - for node in native_field(project, field)?.try_iter()? { + let collections: [Bound<'py, PyAny>; 3] = [ + project_reads::models(project)?, + project_reads::seeds(project)?, + project_reads::functions(project)?, + ]; + for collection in collections { + for node in collection.try_iter()? { nodes.push(node?); } } @@ -151,7 +157,7 @@ fn model_upstream_names( selected_names: &HashSet, ) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); - for reference in native_field(model, "references")?.try_iter()? { + for reference in model_reads::references(model)?.try_iter()? { let reference = reference?; let kind = native_field(&reference, "ref_kind")?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -189,7 +195,7 @@ fn metadata_name_filter( let models = by_name(project, "models")?; let seeds = by_name(project, "seeds")?; let sources = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; sources.set_item(native_field(&entry, "name")?, entry)?; } @@ -271,7 +277,7 @@ fn prefetch_schemas<'py>( for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); } - for source in native_field(&request.project, "sources")?.try_iter()? { + for source in project_reads::sources(&request.project)?.try_iter()? { let source = source?; let entry = native_field(&source, "source_entry")?; if (!relevant.is_none() && !relevant.contains(native_field(&source, "key")?)?) @@ -466,7 +472,7 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> .collect::>()?; let mut all_selected = true; let mut any_migration = false; - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !names.contains(&model_reads::model_name(&model)?) { all_selected = false; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 7c63930fa3..6f0bcca6b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -1,6 +1,7 @@ //! Warehouse source relation listing and column gathering for planning. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -19,7 +20,7 @@ fn entries_of<'py>( ) -> PyResult>> { match source_entries.filter(|entries| !entries.is_none()) { Some(entries) => entries.try_iter()?.collect(), - None => native_field(project, "sources")? + None => project_reads::sources(project)? .try_iter()? .map(|source| native_field(&source?, "source_entry")) .collect(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b0f40a579d..502d7bf9e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -5,6 +5,7 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::native_host::host; @@ -27,7 +28,7 @@ use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; if !native_field(&entry, "loader")?.is_none() { names.insert(native_field(&entry, "name")?.extract()?); @@ -87,7 +88,7 @@ fn selected_managed_refs( managed: &BTreeSet, ) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected.contains(model_reads::model_key(&model)?)? { continue; @@ -95,7 +96,7 @@ fn selected_managed_refs( names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if selected.contains(native_field(&function, "key")?)? { names.extend(sql_source_reads( @@ -104,7 +105,7 @@ fn selected_managed_refs( )?); } } - for audit in native_field(project, "audits")?.try_iter()? { + for audit in project_reads::audits(project)?.try_iter()? { let audit = audit?; let deps = native_field(&audit, "scope_deps")?; if deps.is_truthy()? { @@ -309,7 +310,7 @@ pub(crate) fn build_source_read_map<'py>( let target = resolve_target(project_config, local_config, target_name)?; let effective_vars = native_field(project, "effective_vars")?; let result = source_map.copy()?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let name: String = native_field(&native_field(&source, "source_entry")?, "name")?.extract()?; @@ -344,7 +345,7 @@ pub(crate) fn with_declared_source_reads<'py>( return Ok(source_map.clone()); } let result = source_map.copy()?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; let name: String = native_field(&entry, "name")?.extract()?; if missing.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index aacf69c846..d41bb47bc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; @@ -96,7 +97,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( let read_map = native_field(relations, "source_read_map")?; let listed = native_field(relations, "listed_source_names")?; let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let name: String = native_field(&source, "name")?.extract()?; let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { @@ -173,7 +174,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; sources.set_item(native_field(&source, "name")?, source)?; } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 29c810f9ed..91579426fe 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1276,3 +1276,13 @@ class SetOperationStarRuleTestCase: model_sql: str expected_codes: tuple[str, ...] expected_distinct_lines: int + + +@dataclass(frozen=True) +class CompiledViewFallbacksTestCase: + """One command run over a natively compiled project.""" + + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_fallbacks: int diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 6ac9989863..81ad095b0b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1011,6 +1011,41 @@ def typed_config_reads(*, project_dir: Path, extra_header: str) -> int: return native_module.compile_stage_work_info(reset=True).get("typed_config_reads", 0) +def compiled_view_fallbacks(*, project_dir: Path, command: tuple[str, ...]) -> tuple[int, int]: + """The exit code of `command` over a native project and its reads that missed a native view.""" + + project_dir.mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "main"\n', + encoding="utf-8", + ) + models_dir: Path = project_dir / "models" + models_dir.mkdir() + (models_dir / "orders.sql").write_text( + 'MODEL (description "Orders.", materialized table, contract enforced, ' + 'columns (order_id (type "INTEGER"), loaded_at (type "INTEGER")));\n\n' + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(1 AS INTEGER) AS loaded_at\n", + encoding="utf-8", + ) + (models_dir / "daily_orders.sql").write_text( + 'MODEL (description "Orders per day.", materialized table);\n\n' + 'SELECT order_id, loaded_at FROM __ref("orders")\n', + encoding="utf-8", + ) + (models_dir / "order_view.sql").write_text( + 'MODEL (description "Order ids.", materialized view);\n\n' + 'SELECT order_id FROM __ref("daily_orders")\n', + encoding="utf-8", + ) + _ = native_module.compile_stage_work_info(reset=True) + exit_code: int = main(["--project-dir", str(project_dir), *command]) + fallbacks: int = native_module.compile_stage_work_info(reset=True).get( + "compiled_view_fallbacks", 0 + ) + return exit_code, fallbacks + + FORMAT_CACHE_UNFORMATTED_MODELS: dict[str, str] = { "orders.sql": ( 'MODEL (description "Orders placed by customers.", materialized table);\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py new file mode 100644 index 0000000000..8764dd4332 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py @@ -0,0 +1,43 @@ +"""Compile and plan read a native project's models through its native views.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + CompiledViewFallbacksTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import compiled_view_fallbacks + + +@pytest.mark.parametrize( + "test_case", + [ + CompiledViewFallbacksTestCase( + description="compiling reads every model natively", + command=("compile", "--json", "--no-cache"), + expected_exit_code=0, + expected_fallbacks=0, + ), + CompiledViewFallbacksTestCase( + description="planning reads every model natively", + command=("plan", "--json", "--no-cache"), + expected_exit_code=0, + expected_fallbacks=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_native_project_when_running_then_no_model_read_falls_back_to_python( + test_case: CompiledViewFallbacksTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code, fallbacks = compiled_view_fallbacks( + project_dir=tmp_path / "project", command=test_case.command + ) + _ = capsys.readouterr() + + assert (exit_code, fallbacks) == (test_case.expected_exit_code, test_case.expected_fallbacks) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 29a9ea2709..73008e7b38 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -43,3 +43,10 @@ class RichLineageAnalysisScalingTestCase: class ExpectedCountTestCase: description: str expected_count: int + + +@dataclass(frozen=True) +class PythonRecordFallbackTestCase: + description: str + expected_upstream_resource: str + expected_fallbacks_counted: bool diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py b/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py new file mode 100644 index 0000000000..c32c11959f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py @@ -0,0 +1,53 @@ +"""Python-built compiled models are read by attribute, and each such read is counted.""" + +from __future__ import annotations + +import pytest + +import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage +from sqlbuild.compiler.compile.types import CompiledModelView +from tests.unit.src.sqlbuild.compiler.lineage._test_types import PythonRecordFallbackTestCase +from tests.unit.src.sqlbuild.compiler.lineage.helpers import ( + make_compiled_model, + make_compiled_project, + project_column_lineage, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonRecordFallbackTestCase( + description="lineage over Python-built models reads them by attribute", + expected_upstream_resource="orders", + expected_fallbacks_counted=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_built_models_when_building_lineage_then_reads_fall_back_and_are_counted( + test_case: PythonRecordFallbackTestCase, +) -> None: + models: tuple[CompiledModelView, ...] = ( + make_compiled_model( + name="orders", query_sql="SELECT 1 AS order_id", inferred_columns=("order_id",) + ), + make_compiled_model( + name="orders_out", + query_sql='SELECT order_id FROM __ref("orders")', + inferred_columns=("order_id",), + ), + ) + _ = native.compile_stage_work_info(reset=True) + lineage: ProjectColumnLineage | None = project_column_lineage( + project=make_compiled_project(models=models) + ) + fallbacks: int = native.compile_stage_work_info(reset=True).get("compiled_view_fallbacks", 0) + + assert lineage is not None + upstream: str = lineage.models["orders_out"].columns[0].upstream_columns[0].resource_name + assert (upstream, fallbacks > 0) == ( + test_case.expected_upstream_resource, + test_case.expected_fallbacks_counted, + )