refactor(native): read remaining planner models and project collections natively

Plan entries, phases, identity metadata, scenarios, selectors and the
audit, seed, test, display and clone command plans read a model's
name, destination, key, schema entry, config, references and enum
columns, and iterate the project's models, sources, seeds, functions,
audits and SQL tests, through the native view helpers. A named
collection lookup serves the callers that pick the collection by name.

Author: kvlonge

crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs

@@ -314,3 +314,12 @@ pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>
     }
     native_field(model, "deps")
 }
+
+/// The model's enum columns by name.
+pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
+    let py: Python<'py> = model.py();
+    if let Some(view) = native_model(model)? {
+        return Ok(view.enum_columns(py)?.into_bound(py).into_any());
+    }
+    native_field(model, "enum_columns")
+}

crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs

@@ -6,6 +6,12 @@ use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBA
 use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView;
 use crate::bindings::_helpers::records::fields::field as native_field;
 
+const MODELS_FIELD: &str = "models";
+const SOURCES_FIELD: &str = "sources";
+const SEEDS_FIELD: &str = "seeds";
+const FUNCTIONS_FIELD: &str = "functions";
+const AUDITS_FIELD: &str = "audits";
+
 /// The native view behind a project object; any other object is a Python-built record, counted.
 #[inline]
 fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult<Option<&'a NativeProjectView>> {
@@ -22,7 +28,7 @@ pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, Py
     if let Some(view) = native_project(project)? {
         return Ok(view.models(py).into_bound(py).into_any());
     }
-    native_field(project, "models")
+    native_field(project, MODELS_FIELD)
 }
 
 /// The project's compiled sources.
@@ -31,7 +37,7 @@ pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, P
     if let Some(view) = native_project(project)? {
         return Ok(view.sources(py).into_bound(py));
     }
-    native_field(project, "sources")
+    native_field(project, SOURCES_FIELD)
 }
 
 /// The project's compiled functions.
@@ -40,7 +46,7 @@ pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py,
     if let Some(view) = native_project(project)? {
         return Ok(view.functions(py).into_bound(py));
     }
-    native_field(project, "functions")
+    native_field(project, FUNCTIONS_FIELD)
 }
 
 /// The project's compiled seeds.
@@ -49,7 +55,7 @@ pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyA
     if let Some(view) = native_project(project)? {
         return Ok(view.seeds(py).into_bound(py));
     }
-    native_field(project, "seeds")
+    native_field(project, SEEDS_FIELD)
 }
 
 /// The project's compiled audits.
@@ -58,5 +64,29 @@ pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, Py
     if let Some(view) = native_project(project)? {
         return Ok(view.audits(py).into_bound(py));
     }
-    native_field(project, "audits")
+    native_field(project, AUDITS_FIELD)
+}
+
+/// The project's resource collection named `field`; any other field is read by attribute.
+pub(crate) fn collection<'py>(
+    project: &Bound<'py, PyAny>,
+    field: &str,
+) -> PyResult<Bound<'py, PyAny>> {
+    match field {
+        MODELS_FIELD => models(project),
+        SOURCES_FIELD => sources(project),
+        SEEDS_FIELD => seeds(project),
+        FUNCTIONS_FIELD => functions(project),
+        AUDITS_FIELD => audits(project),
+        _ => native_field(project, field),
+    }
+}
+
+/// The project's compiled SQL tests.
+pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
+    let py: Python<'py> = project.py();
+    if let Some(view) = native_project(project)? {
+        return Ok(view.sql_tests(py).into_bound(py));
+    }
+    native_field(project, "sql_tests")
 }

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs

@@ -3,6 +3,7 @@
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
 use crate::bindings::_helpers::boundary::python_calls::keywords;
 use crate::bindings::_helpers::compiled_views::model_reads;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph;
 use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values};
 use crate::bindings::_helpers::planner_entries::models::ModelPlanContext;
@@ -121,13 +122,13 @@ fn resolved_source_entries<'py>(
     let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?;
     let read = PySet::empty(py)?;
     let mut readers: Vec<Bound<'py, PyAny>> = Vec::new();
-    for model in native_field(project, "models")?.try_iter()? {
+    for model in project_reads::models(project)?.try_iter()? {
         let model = model?;
         if recreated_view(&model, &selected)? {
             readers.push(model);
         }
     }
-    for function in native_field(project, "functions")?.try_iter()? {
+    for function in project_reads::functions(project)?.try_iter()? {
         let function = function?;
         if selected.contains(native_field(&function, "key")?)? {
             readers.push(function);
@@ -157,14 +158,11 @@ fn clone_resolution<'py>(
     (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>),
 ) -> PyResult<CloneResolution<'py>> {
     let py = project.py();
-    let model_locations = object(
-        py,
-        build_model_locations(&native_field(project, "models")?)?,
-    )?;
-    let seed_locations = object(py, build_seed_locations(&native_field(project, "seeds")?)?)?;
+    let model_locations = object(py, build_model_locations(&project_reads::models(project)?)?)?;
+    let seed_locations = object(py, build_seed_locations(&project_reads::seeds(project)?)?)?;
     let function_locations = object(
         py,
-        build_function_locations(&native_field(project, "functions")?)?,
+        build_function_locations(&project_reads::functions(project)?)?,
     )?;
     let source_map = PyDict::new(py);
     for source in native_field(source_project, "sources")?.try_iter()? {
@@ -232,15 +230,15 @@ fn clone_model_entries<'py>(
     let py = project.py();
     let selected = native_field(plan, "selected_keys")?;
     let entries = PyDict::new(py);
-    for model in native_field(project, "models")?.try_iter()? {
+    for model in project_reads::models(project)?.try_iter()? {
         let model = model?;
         if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? {
             continue;
         }
         let materialization_type = materialization_of(&model)?;
         let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? {
             let functions = PyDict::new(py);
-            for function in native_field(project, "functions")?.try_iter()? {
+            for function in project_reads::functions(project)?.try_iter()? {
                 let function = function?;
                 functions.set_item(native_field(&function, "name")?, function)?;
             }
@@ -301,7 +299,7 @@ fn clone_source_entries<'py>(
 ) -> PyResult<Bound<'py, PyAny>> {
     let py = project.py();
     let entries = PyDict::new(py);
-    for source in native_field(project, "sources")?.try_iter()? {
+    for source in project_reads::sources(project)?.try_iter()? {
         let source = source?;
         let entry = native_field(&source, "source_entry")?;
         if !selected.contains(native_field(&source, "key")?)?
@@ -359,7 +357,7 @@ fn clone_function_entries<'py>(
     let py = project.py();
     let selected = native_field(plan, "selected_keys")?;
     let entries = PyDict::new(py);
-    for function in native_field(project, "functions")?.try_iter()? {
+    for function in project_reads::functions(project)?.try_iter()? {
         let function = function?;
         if !selected.contains(native_field(&function, "key")?)? {
             continue;
@@ -445,7 +443,7 @@ fn seed_entries<'py>(
     wanted: &Bound<'py, PyAny>,
 ) -> PyResult<Bound<'py, PyAny>> {
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for seed in native_field(project, "seeds")?.try_iter()? {
+    for seed in project_reads::seeds(project)?.try_iter()? {
         let seed = seed?;
         if wanted.contains(native_field(&seed, field)?)? {
             entries.push(seed_entry(&seed)?);
@@ -460,7 +458,7 @@ fn origin_model_entries<'py>(
 ) -> PyResult<Bound<'py, PyAny>> {
     let py = project.py();
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for model in native_field(project, "models")?.try_iter()? {
+    for model in project_reads::models(project)?.try_iter()? {
         let model = model?;
         if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? {
             continue;
@@ -487,7 +485,7 @@ fn origin_source_entries<'py>(
     let py = project.py();
     let mut keys: Vec<Bound<'py, PyAny>> = Vec::new();
     let mut order: Vec<Bound<'py, PyAny>> = Vec::new();
-    for source in native_field(project, "sources")?.try_iter()? {
+    for source in project_reads::sources(project)?.try_iter()? {
         let source = source?;
         if selected_names.contains(native_field(&source, "name")?)? {
             keys.push(native_field(&source, "key")?);

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs

@@ -3,6 +3,7 @@
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
 use crate::bindings::_helpers::boundary::python_calls::keywords;
 use crate::bindings::_helpers::compiled_views::model_reads;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints;
 use crate::bindings::_helpers::planner_entries::models::RelationsRequest;
 use crate::bindings::_helpers::planner_entries::plan_output::{
@@ -159,7 +160,7 @@ pub(crate) fn build_audit_command_plan<'py>(
 ) -> PyResult<Bound<'py, PyAny>> {
     let py = project.py();
     let materializations = PyDict::new(py);
-    for model in native_field(project, "models")?.try_iter()? {
+    for model in project_reads::models(project)?.try_iter()? {
         let model = model?;
         materializations.set_item(
             model_reads::name_object(&model)?,
@@ -200,7 +201,7 @@ fn called_function_keys<'py>(
 ) -> PyResult<Bound<'py, PySet>> {
     let py: Python<'py> = project.py();
     let functions: Bound<'py, PyDict> = PyDict::new(py);
-    for function in native_field(project, "functions")?.try_iter()? {
+    for function in project_reads::functions(project)?.try_iter()? {
         let function = function?;
         functions.set_item(native_field(&function, "key")?, function)?;
     }
@@ -254,7 +255,7 @@ pub(crate) fn build_test_command_plan<'py>(
     )?;
     let (entries, warnings) = (selected.entries, selected.warnings);
     let mut case_names: BTreeSet<String> = BTreeSet::new();
-    for test in native_field(&project, "sql_tests")?.try_iter()? {
+    for test in project_reads::sql_tests(&project)?.try_iter()? {
         let test = test?;
         let case_name = native_field(&test, "case_name")?;
         if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? {
@@ -271,7 +272,7 @@ pub(crate) fn build_test_command_plan<'py>(
     }
     let function_keys = called_function_keys(&project, function_keys)?;
     let mut functions: Vec<Bound<'py, PyAny>> = Vec::new();
-    for function in native_field(&project, "functions")?.try_iter()? {
+    for function in project_reads::functions(&project)?.try_iter()? {
         let function = function?;
         if !function_keys.contains(native_field(&function, "key")?)? {
             continue;
@@ -331,7 +332,7 @@ pub(crate) fn build_seed_command_plan<'py>(
     };
     let selected_keys = native_field(scope, "selected_keys")?;
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for seed in native_field(project, "seeds")?.try_iter()? {
+    for seed in project_reads::seeds(project)?.try_iter()? {
         let seed = seed?;
         if !selected_keys.contains(native_field(&seed, "key")?)? {
             continue;

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs

@@ -1,6 +1,7 @@
 //! Display-only plans for external plan surfaces when warehouse planning cannot resolve.
 
 use crate::bindings::_helpers::compiled_views::model_reads;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values};
 use crate::bindings::_helpers::planner_entries::values::{
     boolean, native, none, planner_model, planner_type, string,
@@ -138,7 +139,7 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>(
             .collect::<PyResult<Vec<_>>>()?,
     )?;
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for model in native_field(project, "models")?.try_iter()? {
+    for model in project_reads::models(project)?.try_iter()? {
         let model = model?;
         if selected.contains(model_reads::name_object(&model)?)? {
             entries.push(display_entry(&model, full_refresh)?);

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs

@@ -3,6 +3,7 @@
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
 use crate::bindings::_helpers::boundary::python_calls::keywords;
 use crate::bindings::_helpers::compiled_views::model_reads;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::discovery::guidance::microbatch_guidance;
 use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation;
 use crate::bindings::_helpers::planner_changes::detect::detect_model_changes;
@@ -426,7 +427,7 @@ pub(crate) fn plan_entries<'py>(
         runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?,
     )?;
     let functions_by_name = PyDict::new(py);
-    for function in native_field(&project, "functions")?.try_iter()? {
+    for function in project_reads::functions(&project)?.try_iter()? {
         let function = function?;
         functions_by_name.set_item(native_field(&function, "name")?, function)?;
     }

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs

@@ -2,6 +2,7 @@
 
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
 use crate::bindings::_helpers::boundary::python_calls::keywords;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps;
 use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs;
 use crate::bindings::_helpers::planner_entries::values::{
@@ -104,7 +105,7 @@ fn seed_entries<'py>(
     let metadata_jsons = native_field(extras, "seed_metadata_jsons")?;
     let plan_reasons = native_field(extras, "seed_plan_reasons")?;
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for seed in native_field(project, "seeds")?.try_iter()? {
+    for seed in project_reads::seeds(project)?.try_iter()? {
         let seed = seed?;
         if !selected_keys.contains(native_field(&seed, "key")?)? {
             continue;
@@ -404,7 +405,7 @@ fn function_entries<'py>(
     let functions = native_field(changes, "functions")?;
     let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?;
     let mut entries: Vec<Bound<'py, PyAny>> = Vec::new();
-    for function in native_field(project, "functions")?.try_iter()? {
+    for function in project_reads::functions(project)?.try_iter()? {
         let function = function?;
         if !selected_keys.contains(native_field(&function, "key")?)? {
             continue;
@@ -464,7 +465,7 @@ pub(super) fn selected_test_entries<'py>(
 ) -> PyResult<SelectedTests<'py>> {
     let py = project.py();
     let mut selected: Vec<Bound<'py, PyAny>> = Vec::new();
-    for test in native_field(project, "sql_tests")?.try_iter()? {
+    for test in project_reads::sql_tests(project)?.try_iter()? {
         let test = test?;
         if !test_selected(&test, selected_keys, selection)? {
             continue;

crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs

@@ -2,6 +2,7 @@
 
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
 use crate::bindings::_helpers::boundary::python_calls::keywords;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::planner_entries::models::{
     PlannerRelationsContext, RelationsRequest,
 };
@@ -31,7 +32,7 @@ fn selected_audit_relation_keys<'py>(
 ) -> PyResult<Bound<'py, PyAny>> {
     let py = project.py();
     let keys = PySet::empty(py)?;
-    for audit in native_field(project, "audits")?.try_iter()? {
+    for audit in project_reads::audits(project)?.try_iter()? {
         let audit = audit?;
         if audit_is_selected(&audit, selected_keys)? {
             for dependency in native_field(&audit, "scope_deps")?.try_iter()? {
@@ -99,12 +100,12 @@ pub(crate) fn planner_relations_context<'py>(
     let selected_keys = native_field(&scope, "selected_keys")?;
     let mut model_locations = object(
         py,
-        build_model_locations(&native_field(&project, "models")?)?,
+        build_model_locations(&project_reads::models(&project)?)?,
     )?;
-    let mut seed_locations = object(py, build_seed_locations(&native_field(&project, "seeds")?)?)?;
+    let mut seed_locations = object(py, build_seed_locations(&project_reads::seeds(&project)?)?)?;
     let function_locations = object(
         py,
-        build_function_locations(&native_field(&project, "functions")?)?,
+        build_function_locations(&project_reads::functions(&project)?)?,
     )?;
     let deferred_locations = native_field(&deferral, "deferred_locations")?;
     if !deferred_locations.is_none() {

crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs

@@ -20,6 +20,7 @@ use sqlbuild_analysis::planner_identity::main::topological_order::topological_or
 use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream};
 
 use crate::bindings::_helpers::boundary::python_calls::KeywordArguments;
+use crate::bindings::_helpers::compiled_views::project_reads;
 use crate::bindings::_helpers::graph::algorithms::{closure, inverted};
 use crate::bindings::_helpers::graph::keys::KeyCache;
 use crate::bindings::_helpers::planner_identity::hashing::graph_key;
@@ -89,8 +90,14 @@ pub(crate) fn build_execution_upstream_deps<'py>(
 ) -> PyResult<Bound<'py, PyDict>> {
     let py: Python<'py> = project.py();
     let upstream: Bound<'py, PyDict> = PyDict::new(py);
-    for collection in ["models", "sources", "seeds", "functions"] {
-        for resource in native_field(project, collection)?.try_iter()? {
+    let collections: [Bound<'py, PyAny>; 4] = [
+        project_reads::models(project)?,
+        project_reads::sources(project)?,
+        project_reads::seeds(project)?,
+        project_reads::functions(project)?,
+    ];
+    for collection in collections {
+        for resource in collection.try_iter()? {
             let resource: Bound<'py, PyAny> = resource?;
             upstream.set_item(
                 native_field(&resource, "key")?,
@@ -103,7 +110,7 @@ pub(crate) fn build_execution_upstream_deps<'py>(
             )?;
         }
     }
-    for test in native_field(project, "sql_tests")?.try_iter()? {
+    for test in project_reads::sql_tests(project)?.try_iter()? {
         let test: Bound<'py, PyAny> = test?;
         let test_key: Bound<'py, PyAny> = native_field(&test, "key")?;
         let scope_deps: Bound<'py, PyAny> = native_field(&test, "scope_deps")?;
@@ -172,7 +179,7 @@ pub(crate) fn build_execution_edge_origins<'py>(
 ) -> PyResult<Bound<'py, PyDict>> {
     let py: Python<'py> = project.py();
     let origins: Bound<'py, PyDict> = PyDict::new(py);
-    for test in native_field(project, "sql_tests")?.try_iter()? {
+    for test in project_reads::sql_tests(project)?.try_iter()? {
         let test: Bound<'py, PyAny> = test?;
         let test_name: String = native_field(&test, "name")?.extract()?;
         for target_key in native_field(&test, "scope_deps")?.try_iter()? {