From 378ad25ef7b357f1771b9d6a1b3476e36694bbf1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:59:08 +0100 Subject: [PATCH 1/4] refactor(native): read compiled models through one counted native accessor `model_reads::native_model` returns the native view behind a model object and counts every other object, a Python-built record that is read by attribute instead, in the `compiled_view_fallbacks` stage counter. Every model read helper goes through it. An integration test shows compile and plan of a native project never fall back, and a unit test shows Python-built models are still read, and counted. --- .../_helpers/compile_driver/stage_work.rs | 2 + .../_helpers/compiled_views/model_reads.rs | 72 +++++++++++-------- .../sqlbuild/cli/commands/main/_test_types.py | 10 +++ .../src/sqlbuild/cli/commands/main/helpers.py | 35 +++++++++ .../commands/main/test_compiled_view_reads.py | 43 +++++++++++ .../sqlbuild/compiler/lineage/_test_types.py | 7 ++ .../lineage/test_python_record_reads.py | 53 ++++++++++++++ 7 files changed, 192 insertions(+), 30 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py create mode 100644 tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs index 88d543f7fa..12cac440d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -34,6 +34,8 @@ pub(crate) const NATIVE_HEADER_COLUMNS: &str = "native_header_columns"; pub(crate) const PARSED_HEADER_COLUMNS: &str = "parsed_header_columns"; /// Config values converted into typed nodes for a key a reader asked for. pub(crate) const TYPED_CONFIG_READS: &str = "typed_config_reads"; +/// Reads of a model or project that is not a native compiled view, read by attribute instead. +pub(crate) const COMPILED_VIEW_FALLBACKS: &str = "compiled_view_fallbacks"; /// Compile stage runs since the last reset, by stage. #[pyclass] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index 752cddb036..60a2d64dfe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -2,6 +2,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ DeclaredRow, LineageInput, NativeCompiledModel, }; @@ -27,10 +28,22 @@ const TRANSFORM_VALUES: [&str; 6] = [ ]; const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; +/// The native view behind a model object; any other object is a Python-built record, counted. +#[inline] +pub(crate) fn native_model<'a>( + model: &'a Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(view) = model.cast::() { + return Ok(Some(view.get())); + } + count_stage(model.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} + /// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().inferred_rows(model.py()); + if let Some(view) = native_model(model)? { + return view.inferred_rows(model.py()); } let rows: Bound<'_, PyAny> = native_field(model, "inferred_rows")?; if rows.is_none() { @@ -56,24 +69,23 @@ fn column_names(rows: Vec) -> Vec { /// Each declared column of the attached schema as `(name, type)`, `None` without a schema. pub(crate) fn declared_rows(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().schema_column_rows(model.py()); + if let Some(view) = native_model(model)? { + return view.schema_column_rows(model.py()); } native_field(model, "schema_column_rows")?.extract() } /// Whether the attached schema declares dynamic column families. pub(crate) fn dynamic_columns_declared(model: &Bound<'_, PyAny>) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().dynamic_columns_declared(model.py()); + if let Some(view) = native_model(model)? { + return view.dynamic_columns_declared(model.py()); } native_field(model, "has_dynamic_columns")?.is_truthy() } /// `(has star, star resolved)` of the model's analysed query. pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { - if let Ok(view) = model.downcast::() { - let view: &NativeCompiledModel = view.get(); + if let Some(view) = native_model(model)? { return Ok(( view.fast_lineage_has_star(model.py())?, view.fast_lineage_star_resolved(model.py())?, @@ -87,8 +99,8 @@ pub(crate) fn star_flags(model: &Bound<'_, PyAny>) -> PyResult<(bool, bool)> { /// Output columns whose types semantic recovery forgot, in any order. pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().unchecked_columns().to_vec()); + if let Some(view) = native_model(model)? { + return Ok(view.unchecked_columns().to_vec()); } native_field(model, "unchecked_output_columns")? .try_iter()? @@ -98,8 +110,8 @@ pub(crate) fn unchecked_columns(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().lineage_inputs(model.py())?.unwrap_or_default()); + if let Some(view) = native_model(model)? { + return Ok(view.lineage_inputs(model.py())?.unwrap_or_default()); } let columns: Bound<'_, PyAny> = native_field(model, "fast_lineage_columns")?; if !columns.is_truthy()? { @@ -123,16 +135,16 @@ pub(crate) fn lineage_inputs(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().binding_validated(model.py()); + if let Some(view) = native_model(model)? { + return view.binding_validated(model.py()); } native_field(model, "binding_validated")?.is_truthy() } /// Whether the model's dynamic pivot proof proved its output columns. pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().output_proven(model.py()); + if let Some(view) = native_model(model)? { + return view.output_proven(model.py()); } let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; Ok(!proof.is_none() && native_field(&proof, "output_proven")?.is_truthy()?) @@ -140,8 +152,8 @@ pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { /// Each lineage output with its transform kind and confidence values, `None` without lineage. pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - let Some(rows) = view.get().lineage_rows(model.py())? else { + if let Some(view) = native_model(model)? { + let Some(rows) = view.lineage_rows(model.py())? else { return Ok(None); }; let mut facts: Vec = Vec::with_capacity(rows.len()); @@ -190,8 +202,8 @@ fn coded(values: &[&str], code: u8) -> PyResult { /// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { - if let Ok(view) = model.downcast::() { - return view.get().proven_inputs(model.py()); + if let Some(view) = native_model(model)? { + return view.proven_inputs(model.py()); } let proof: Bound<'_, PyAny> = native_field(model, "dynamic_column_contract")?; if proof.is_none() || !native_field(&proof, "output_proven")?.is_truthy()? { @@ -202,40 +214,40 @@ pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - if let Ok(view) = model.downcast::() { - return view.get().name(model.py()); + if let Some(view) = native_model(model)? { + return view.name(model.py()); } native_field(model, "name")?.extract() } /// The model's compiled query SQL. pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().query_sql(model.py())?.into_any()); + if let Some(view) = native_model(model)? { + return Ok(view.query_sql(model.py())?.into_any()); } native_field(model, "query_sql") } /// The model's `CompiledObjectKey`. pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().key(model.py())?.into_bound(model.py())); + if let Some(view) = native_model(model)? { + return Ok(view.key(model.py())?.into_bound(model.py())); } native_field(model, "key") } /// The model's destination `CompiledRelationLocation`. pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - return Ok(view.get().destination(model.py())?.into_bound(model.py())); + if let Some(view) = native_model(model)? { + return Ok(view.destination(model.py())?.into_bound(model.py())); } native_field(model, "destination") } /// The model's name as a Python string. pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - if let Ok(view) = model.downcast::() { - let name: String = view.get().name(model.py())?; + if let Some(view) = native_model(model)? { + let name: String = view.name(model.py())?; return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); } native_field(model, "name") diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 29c810f9ed..91579426fe 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1276,3 +1276,13 @@ class SetOperationStarRuleTestCase: model_sql: str expected_codes: tuple[str, ...] expected_distinct_lines: int + + +@dataclass(frozen=True) +class CompiledViewFallbacksTestCase: + """One command run over a natively compiled project.""" + + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_fallbacks: int diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 6ac9989863..81ad095b0b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1011,6 +1011,41 @@ def typed_config_reads(*, project_dir: Path, extra_header: str) -> int: return native_module.compile_stage_work_info(reset=True).get("typed_config_reads", 0) +def compiled_view_fallbacks(*, project_dir: Path, command: tuple[str, ...]) -> tuple[int, int]: + """The exit code of `command` over a native project and its reads that missed a native view.""" + + project_dir.mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connections.local]\ndatabase = ":memory:"\n\n[targets.dev]\nschema = "main"\n', + encoding="utf-8", + ) + models_dir: Path = project_dir / "models" + models_dir.mkdir() + (models_dir / "orders.sql").write_text( + 'MODEL (description "Orders.", materialized table, contract enforced, ' + 'columns (order_id (type "INTEGER"), loaded_at (type "INTEGER")));\n\n' + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(1 AS INTEGER) AS loaded_at\n", + encoding="utf-8", + ) + (models_dir / "daily_orders.sql").write_text( + 'MODEL (description "Orders per day.", materialized table);\n\n' + 'SELECT order_id, loaded_at FROM __ref("orders")\n', + encoding="utf-8", + ) + (models_dir / "order_view.sql").write_text( + 'MODEL (description "Order ids.", materialized view);\n\n' + 'SELECT order_id FROM __ref("daily_orders")\n', + encoding="utf-8", + ) + _ = native_module.compile_stage_work_info(reset=True) + exit_code: int = main(["--project-dir", str(project_dir), *command]) + fallbacks: int = native_module.compile_stage_work_info(reset=True).get( + "compiled_view_fallbacks", 0 + ) + return exit_code, fallbacks + + FORMAT_CACHE_UNFORMATTED_MODELS: dict[str, str] = { "orders.sql": ( 'MODEL (description "Orders placed by customers.", materialized table);\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py new file mode 100644 index 0000000000..8764dd4332 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compiled_view_reads.py @@ -0,0 +1,43 @@ +"""Compile and plan read a native project's models through its native views.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + CompiledViewFallbacksTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import compiled_view_fallbacks + + +@pytest.mark.parametrize( + "test_case", + [ + CompiledViewFallbacksTestCase( + description="compiling reads every model natively", + command=("compile", "--json", "--no-cache"), + expected_exit_code=0, + expected_fallbacks=0, + ), + CompiledViewFallbacksTestCase( + description="planning reads every model natively", + command=("plan", "--json", "--no-cache"), + expected_exit_code=0, + expected_fallbacks=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_native_project_when_running_then_no_model_read_falls_back_to_python( + test_case: CompiledViewFallbacksTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code, fallbacks = compiled_view_fallbacks( + project_dir=tmp_path / "project", command=test_case.command + ) + _ = capsys.readouterr() + + assert (exit_code, fallbacks) == (test_case.expected_exit_code, test_case.expected_fallbacks) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index 29a9ea2709..73008e7b38 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -43,3 +43,10 @@ class RichLineageAnalysisScalingTestCase: class ExpectedCountTestCase: description: str expected_count: int + + +@dataclass(frozen=True) +class PythonRecordFallbackTestCase: + description: str + expected_upstream_resource: str + expected_fallbacks_counted: bool diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py b/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py new file mode 100644 index 0000000000..c32c11959f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_python_record_reads.py @@ -0,0 +1,53 @@ +"""Python-built compiled models are read by attribute, and each such read is counted.""" + +from __future__ import annotations + +import pytest + +import sqlbuild._native as native +from sqlbuild._native import ProjectColumnLineage +from sqlbuild.compiler.compile.types import CompiledModelView +from tests.unit.src.sqlbuild.compiler.lineage._test_types import PythonRecordFallbackTestCase +from tests.unit.src.sqlbuild.compiler.lineage.helpers import ( + make_compiled_model, + make_compiled_project, + project_column_lineage, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonRecordFallbackTestCase( + description="lineage over Python-built models reads them by attribute", + expected_upstream_resource="orders", + expected_fallbacks_counted=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_built_models_when_building_lineage_then_reads_fall_back_and_are_counted( + test_case: PythonRecordFallbackTestCase, +) -> None: + models: tuple[CompiledModelView, ...] = ( + make_compiled_model( + name="orders", query_sql="SELECT 1 AS order_id", inferred_columns=("order_id",) + ), + make_compiled_model( + name="orders_out", + query_sql='SELECT order_id FROM __ref("orders")', + inferred_columns=("order_id",), + ), + ) + _ = native.compile_stage_work_info(reset=True) + lineage: ProjectColumnLineage | None = project_column_lineage( + project=make_compiled_project(models=models) + ) + fallbacks: int = native.compile_stage_work_info(reset=True).get("compiled_view_fallbacks", 0) + + assert lineage is not None + upstream: str = lineage.models["orders_out"].columns[0].upstream_columns[0].resource_name + assert (upstream, fallbacks > 0) == ( + test_case.expected_upstream_resource, + test_case.expected_fallbacks_counted, + ) From b6f55c656a6eac7bca73594cfea70ae34e3088fc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:50:21 +0100 Subject: [PATCH 2/4] refactor(native): read resolution and strategy models through native views Planner resolution, strategy, scope and change detection read a model's schema entry, config, references, inferred columns, relative path, name and dependency rows, and iterate the project's models, sources, seeds, functions and audits, through the native view helpers instead of attribute reads. Reads that serve every resource kind keep the shared attribute dispatch. --- .../bindings/_helpers/compiled_views/mod.rs | 1 + .../_helpers/compiled_views/model_reads.rs | 55 +++++++++++++++- .../_helpers/compiled_views/project_reads.rs | 62 +++++++++++++++++++ .../_helpers/planner_changes/detect.rs | 7 ++- .../_helpers/planner_changes/renames.rs | 2 +- .../bindings/_helpers/planner_scope/scope.rs | 8 ++- .../_helpers/planner_scope/sources.rs | 3 +- .../bindings/_helpers/planner_scope/values.rs | 3 +- .../planner_strategy/audit_entries.rs | 3 +- .../planner_strategy/fixture_context.rs | 16 +++-- .../planner_strategy/fixture_metadata.rs | 3 +- .../planner_strategy/fixture_reads.rs | 4 +- .../_helpers/planner_strategy/lineage.rs | 3 +- .../planner_strategy/model_entries.rs | 10 +-- 14 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs index a8c2429d4e..ecdb130732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod inputs_view; pub(crate) mod model_input_view; pub(crate) mod model_reads; pub(crate) mod project_host; +pub(crate) mod project_reads; pub(crate) mod project_view; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index 60a2d64dfe..9419287724 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -1,6 +1,6 @@ //! A compiled model's analysis outcomes as Rust data: views read their owner, records their fields. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; use crate::bindings::_helpers::compiled_views::compiled_model_view::{ @@ -252,3 +252,56 @@ pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.schema_entry(py)?.into_bound(py)); + } + native_field(model, "schema_entry") +} + +/// The model's `CompileModelConfig`. +pub(crate) fn config<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.config(py)?.into_bound(py)); + } + native_field(model, "config") +} + +/// The model's `CompileSqlReference` tuple. +pub(crate) fn references<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.references(py)?.into_bound(py).into_any()); + } + native_field(model, "references") +} + +/// The model file's relative path. +pub(crate) fn relative_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.relative_path(py)?.into_bound(py)); + } + native_field(model, "relative_path") +} + +/// The model's `InferredColumn` tuple, or `None`. +pub(crate) fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.inferred_columns(py)?.into_bound(py)); + } + native_field(model, "inferred_columns") +} + +/// `(resource type, name)` of each dependency. +pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult> { + if let Some(view) = native_model(model)? { + return view.dep_rows(model.py()); + } + native_field(model, "dep_rows")?.extract() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs new file mode 100644 index 0000000000..dc81541f2f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs @@ -0,0 +1,62 @@ +//! A compiled project's resource collections, read from its native view. + +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; + +use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBACKS, count_stage}; +use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The native view behind a project object; any other object is a Python-built record, counted. +#[inline] +fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult> { + if let Ok(view) = project.cast::() { + return Ok(Some(view.get())); + } + count_stage(project.py(), COMPILED_VIEW_FALLBACKS, 1)?; + Ok(None) +} + +/// The project's compiled models, in compile order. +pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.models(py).into_bound(py).into_any()); + } + native_field(project, "models") +} + +/// The project's compiled sources. +pub(crate) fn sources<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sources(py).into_bound(py)); + } + native_field(project, "sources") +} + +/// The project's compiled functions. +pub(crate) fn functions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.functions(py).into_bound(py)); + } + native_field(project, "functions") +} + +/// The project's compiled seeds. +pub(crate) fn seeds<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.seeds(py).into_bound(py)); + } + native_field(project, "seeds") +} + +/// The project's compiled audits. +pub(crate) fn audits<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.audits(py).into_bound(py)); + } + native_field(project, "audits") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index bac3ace9cd..3aeb87e01c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::planner_changes::results::native_enum; @@ -107,7 +108,7 @@ pub(crate) fn detect_changes<'py>( let query_change_tracking: bool = native_field(&native_field(project, "settings")?, "query_change_tracking")?.is_truthy()?; let dialect: Option = native_field(snapshot, "column_dialect")?.extract()?; - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = native_field(&function, "name")?; let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; @@ -248,7 +249,7 @@ pub(crate) fn scope_model_changes<'py>( } }; let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( - &native_field(project, "functions")?, + &project_reads::functions(project)?, native_field(project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -530,7 +531,7 @@ fn model_schema_findings<'py>( for column in &validity { warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); } - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; let mut dynamic: Option> = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index df127086a8..d54dd25725 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -240,7 +240,7 @@ fn changed_referencing_query( continue; } let mut references_models: bool = false; - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { if native_field(&reference?, "ref_kind")?.eq(&ref_kind)? { references_models = true; break; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index 30ae9b2874..3ea5d202b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -1,6 +1,8 @@ //! Planner scope assembly using native graph and selector operations. use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::graph::views::lineage_graph_names; @@ -33,7 +35,7 @@ pub(crate) fn managed_source_upstream_keys<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let mut managed: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if !native_field(&native_field(&source, "source_entry")?, "loader")?.is_none() { managed.insert(native_field(&source, "name")?.extract()?); @@ -119,9 +121,9 @@ pub(crate) fn build_planner_scope<'py>( let origins = build_execution_edge_origins(&project)?.into_any(); let order = topologically_order_keys(&upstream, Some(origins))?.into_any(); let models = PyDict::new(py); - for model in native_field(&project, "models")?.try_iter()? { + for model in project_reads::models(&project)?.try_iter()? { let model = model?; - models.set_item(native_field(&model, "name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let mut reads = args.value("python_read_source_names")?; if reads.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs index eb354739b7..3b789fa489 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs @@ -2,6 +2,7 @@ use sqlbuild_host::entries::models::ExecutorHost; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::algorithms::closure; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; @@ -30,7 +31,7 @@ pub(crate) fn build_source_load_map<'py>( }) .transpose()?; let result = PyDict::new(project.py()); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if let Some(keys) = &relevant && !keys.contains(native_field(&source, "key")?)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs index 0058a12dec..8102e74d73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -1,5 +1,6 @@ //! Python function identity and compiled graph conversion. +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::native_key; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -67,7 +68,7 @@ pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { let mut graph = loader_graph(&native_field(project, "loader_functions")?)?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; graph.sources.push(( native_field(&entry, "name")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 189e4be386..f4dcdde7be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::auditing::render::render_audit_sql; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::views::project_lineage_views; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::audits::{ @@ -229,7 +230,7 @@ pub(crate) fn build_selected_audit_entries<'py>( let project = request.get("project")?; let selected_keys = native_field(&request.get("scope")?, "selected_keys")?; let mut selected: Vec> = Vec::new(); - for audit in native_field(&project, "audits")?.try_iter()? { + for audit in project_reads::audits(&project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, &selected_keys)? { selected.push(audit); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 7f9fb73a19..dc3f0b3088 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -13,6 +13,7 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_strategy::constants::{ MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, }; @@ -53,7 +54,7 @@ pub(crate) fn planning_context( Some(models) => models, None => { let models = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, &model)?; } @@ -67,11 +68,16 @@ pub(crate) fn planning_context( model_fixture_relation(&model)?, ); } - for (collection, entry, kind, source) in [ - ("sources", "source_entry", SOURCE_REFERENCE_KIND, true), - ("seeds", "schema_entry", SEED_REFERENCE_KIND, false), + for (entry, kind, source) in [ + ("source_entry", SOURCE_REFERENCE_KIND, true), + ("schema_entry", SEED_REFERENCE_KIND, false), ] { - for resource in native_field(project, collection)?.try_iter()? { + let resources: Bound<'_, PyAny> = if source { + project_reads::sources(project)? + } else { + project_reads::seeds(project)? + }; + for resource in resources.try_iter()? { let resource = resource?; let entry = native_field(&resource, entry)?; let columns: Vec = columns(&native_field(&entry, "columns")?, false)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index edd894a333..c488412173 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -5,6 +5,7 @@ use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::compiled_views::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_strategy::constants::ENFORCED_CONTRACT; @@ -18,7 +19,7 @@ const CONTRACT_KEY: &str = "contract"; /// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { let inferred: Option> = inferred_rows(model)?; - let schema = native_field(model, "schema_entry")?; + let schema = model_reads::schema_entry(model)?; let declared: Vec = if schema.is_none() { Vec::new() } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 20f5bdbd1f..aacaa6ffab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -62,7 +62,7 @@ pub(crate) fn required_fixture_columns( continue; }; let mut references: Vec = Vec::new(); - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { let reference = reference?; references.push(FixtureReadReference { kind: native_field(&reference, "ref_kind")?.extract()?, @@ -132,7 +132,7 @@ fn model_stars( return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for key in native_field(model, "dep_rows")?.extract::>()? { + for key in model_reads::dep_rows(model)? { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index e25f1f94c8..cf58cc1fd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -86,8 +86,7 @@ impl<'py> ReferenceGraph for ProjectLineage<'py> { fn producer_facts(&self, name: &str) -> PyResult { let producer: Bound<'py, PyAny> = self.models_by_name.get_item(name)?; - let values: Bound<'py, PyAny> = - native_field(&native_field(&producer, "config")?, "values")?; + let values: Bound<'py, PyAny> = native_field(&model_reads::config(&producer)?, "values")?; let capped: bool = match microbatch_limit_config(&values)?.1 { Some(action) => CAP_ACTIONS.contains(&text(&action)?.as_str()), None => false, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 260780475d..009cd9e0b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -195,7 +195,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; if !entry.is_none() { let enforcement: Bound<'py, PyAny> = native_field(&entry, "type_enforcement")?; if !enforcement.is_none() { @@ -310,7 +310,7 @@ fn logical_ddl<'py>( /// The output column names SQL analysis inferred for the model, when it inferred them. fn expected_output_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = model.py(); - let inferred: Bound<'py, PyAny> = native_field(model, "inferred_columns")?; + let inferred: Bound<'py, PyAny> = model_reads::inferred_columns(model)?; if inferred.is_none() { return Ok(none(py)); } @@ -577,7 +577,7 @@ fn contract_fields<'py>( mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?, )?; entry.set_item("contract_columns", schema_columns(model, false)?)?; - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; entry.set_item( "contract_dynamic_columns", if schema_entry.is_none() { @@ -653,7 +653,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( .unwrap_or_else(|| PyTuple::empty(py)); let table_type: Bound<'py, PyAny> = native_field( &native_field( - &native_field(&native_field(model, "config")?, "table_type")?, + &native_field(&model_reads::config(model)?, "table_type")?, "value", )?, "value", @@ -666,7 +666,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let entry: Bound<'py, PyDict> = PyDict::new(py); entry.set_item("key", model_reads::model_key(model)?)?; entry.set_item("name", &name)?; - entry.set_item("relative_path", native_field(model, "relative_path")?)?; + entry.set_item("relative_path", model_reads::relative_path(model)?)?; entry.set_item("destination", model_reads::destination(model)?)?; entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; From 0610be2a9a537c99863b7a80e5636e3d4157c477 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:29:01 +0100 Subject: [PATCH 3/4] refactor(native): read warehouse and migration models through native views Planner warehouse and migration code read a model's name, destination, key, schema entry, config, references and dependency keys, and iterate the project's models, seeds, functions and sources, through the native view helpers. Reads that serve every resource kind keep the shared attribute dispatch. --- .../_helpers/compiled_views/model_reads.rs | 9 +++++++ .../planner_migrations/column_candidates.rs | 2 +- .../_helpers/planner_migrations/discovery.rs | 5 ++-- .../planner_migrations/discovery_events.rs | 5 ++-- .../planner_migrations/discovery_models.rs | 21 +++++++-------- .../planner_migrations/discovery_requests.rs | 9 ++++--- .../planner_migrations/model_entries.rs | 7 ++--- .../planner_migrations/model_planning.rs | 13 +++++----- .../planner_migrations/old_name_entries.rs | 6 ++--- .../planner_migrations/old_name_objects.rs | 5 ++-- .../planner_migrations/old_name_planning.rs | 12 +++++---- .../_helpers/planner_migrations/old_names.rs | 2 +- .../planner_migrations/planning_origin.rs | 26 ++++++++----------- .../_helpers/planner_warehouse/cursors.rs | 5 ++-- .../planner_warehouse/semantic_sources.rs | 5 ++-- .../_helpers/planner_warehouse/snapshot.rs | 18 ++++++++----- .../planner_warehouse/source_deferral.rs | 11 ++++---- .../planner_warehouse/source_tables.rs | 3 ++- 18 files changed, 92 insertions(+), 72 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index 9419287724..4b5874bfa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -305,3 +305,12 @@ pub(crate) fn dep_rows(model: &Bound<'_, PyAny>) -> PyResult(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.deps(py)?.into_bound(py).into_any()); + } + native_field(model, "deps") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index beb4f4ff94..6cca98b4c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -45,7 +45,7 @@ pub(super) fn candidates<'py>( let warehouse = native_field(&snapshot, "existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let declared = PyDict::new(py); - let schema = native_field(&model, "schema_entry")?; + let schema = model_reads::schema_entry(&model)?; if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { let column = column?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index 2cab5a269b..77549f4fac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,6 +1,7 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::matching::{ match_migration_fingerprints, migration_destination_fingerprints, }; @@ -56,7 +57,7 @@ pub(super) fn discover_model_migrations<'py>( let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); for item in &models { - names.set_item(native_field(item, "name")?, item)?; + names.set_item(model_reads::name_object(item)?, item)?; } let event_renames = discovery_events::renames(&runtime, &names, &state)?; for (name, origin) in declared.iter() { @@ -74,7 +75,7 @@ pub(super) fn discover_model_migrations<'py>( } let unbuilt = PyDict::new(py); for item in &models { - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; if !declared_names.contains(&name)? && call_with( &native_field(&state, "named_fingerprint")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 9bec76268f..966ce3f899 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; @@ -110,11 +111,11 @@ pub(super) fn resumed<'py>( let target = native_field(&native_field(runtime, "project")?, "effective_target_name")?; let events = native_field(state, "events")?; for item in models { - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; if !unbuilt.contains(&name)? || !(history(item)? || identity(item)?) { continue; } - let destination = native_field(item, "destination")?; + let destination = model_reads::destination(item)?; let event = newest(&events, &destination)?; if event.is_none() { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 861a44763d..38899cfdf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -2,6 +2,8 @@ use sqlbuild_host::entries::models::UserCodeHost; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; @@ -19,16 +21,13 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - let name: String = native_field(&item, "name")?.extract()?; + let name: String = model_reads::name_object(&item)?.extract()?; let mut upstream: Vec = Vec::new(); - for dependency in native_field(&item, "deps")?.try_iter()? { + for dependency in model_reads::deps(&item)?.try_iter()? { let dependency = dependency?; if native_field(&dependency, "resource_type")?.eq("model")? { upstream.push(native_field(&dependency, "name")?.extract()?); @@ -63,8 +62,8 @@ pub(super) fn triggered( let relevant = relevant.0; let fingerprints = native_field(&native_field(snapshot, "fingerprints")?, "models")?; for item in models { - let name = native_field(item, "name")?; - if relevant.contains(native_field(item, "key")?)? + let name = model_reads::name_object(item)?; + if relevant.contains(model_reads::model_key(item)?)? && !fingerprints.contains(&name)? && !declared.contains(&name)? { @@ -83,7 +82,7 @@ pub(super) fn metadata<'py>( let functions = object( py, build_function_local_hashes( - &native_field(&project, "functions")?, + &project_reads::functions(&project)?, native_field(&project, "sql_analysis_dialect")? .extract::>()? .as_deref(), @@ -102,7 +101,7 @@ pub(super) fn metadata<'py>( let result = PyDict::new(py); for item in models { result.set_item( - native_field(item, "name")?, + model_reads::name_object(item)?, object( py, build_model_version_identity_metadata_json( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index a8bde508ea..b0dd9cf867 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -1,6 +1,7 @@ //! Live candidate inspection and automatic request materialization. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::config; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ @@ -114,10 +115,10 @@ pub(super) fn automatic<'py>( let py = runtime.py(); let mut requests: Vec> = Vec::new(); for item in models { - if !native_field(scope, "selected_keys")?.contains(native_field(item, "key")?)? { + if !native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { continue; } - let name = native_field(item, "name")?; + let name = model_reads::name_object(item)?; let event = matches.1.get_item(&name)?; let candidate = mapping_get(matches.0, &name)?; let origin = if let Some(event) = event { @@ -162,8 +163,8 @@ pub(super) fn warnings<'py>( let py = scope.py(); let selected = PySet::empty(py)?; for item in models { - if native_field(scope, "selected_keys")?.contains(native_field(item, "key")?)? { - selected.add(native_field(item, "name")?)?; + if native_field(scope, "selected_keys")?.contains(model_reads::model_key(item)?)? { + selected.add(model_reads::name_object(item)?)?; } } let ambiguous: BTreeMap = ambiguous.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 344d3ab855..bfda1b78be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; @@ -73,7 +74,7 @@ pub(super) fn decide<'py>( let py = runtime.py(); let request = decision_request.request; let item = request.model.clone(); - let destination = native_field(&item, "destination")?; + let destination = model_reads::destination(&item)?; let origin = decision_request.origin.clone(); let snapshot = decision_request.snapshot.clone(); let state = decision_request.state.clone(); @@ -99,7 +100,7 @@ pub(super) fn decide<'py>( py, RecordId::ModelMigrationPlanEntry, &[ - ("model_name", native_field(&item, "name")?), + ("model_name", model_reads::name_object(&item)?), ("discovery", request.discovery.clone()), ( "decision", @@ -160,7 +161,7 @@ pub(super) fn decide<'py>( } let inherited = if recorded && !applies(&item, &newest, &snapshot)? { native_field(&native_field(&snapshot, "fingerprints")?, "models")? - .call_method1("get", (native_field(&item, "name")?,))? + .call_method1("get", (model_reads::name_object(&item)?,))? } else { empty_version(&item, &fingerprint)? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 453a186d4c..2a90e97ead 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,6 +2,8 @@ use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{ config_node_of, config_value, model_values, }; @@ -97,11 +99,8 @@ fn discover<'py>( let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in native_field( - &runtime_field(&runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(&runtime, |runtime| &runtime.project)?)?.try_iter()? { if let Some(request) = declared(&item?)? { declared_requests.push(request); @@ -117,7 +116,7 @@ fn discover<'py>( for request in &declared_requests { let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; declarations.push(ModelMigrationDeclaration { - model_name: native_field(&request.model, "name")?, + model_name: model_reads::name_object(&request.model)?, origin_location, origin_model, }); @@ -183,7 +182,7 @@ fn finish<'py>( let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { let item = request.model.clone(); - let name = native_field(&item, "name")?; + let name = model_reads::name_object(&item)?; let decided = super::model_entries::decide(&MigrationDecisionRequest { runtime: runtime.clone(), request, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 33e58dcdfe..be08c9830c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; @@ -113,8 +114,7 @@ impl<'py> OldNamePlanner<'py> { &self, item: &Bound<'py, PyAny>, ) -> PyResult>> { - let Some(history) = self.newest(&relation(&native_field(item, "destination")?)?, None)? - else { + let Some(history) = self.newest(&relation(&model_reads::destination(item)?)?, None)? else { return Ok(None); }; if !native_field(&native_field(&history, "move")?, "destination_model")? @@ -324,7 +324,7 @@ impl<'py> OldNamePlanner<'py> { py, RecordId::OldNameViewPlanEntry, &[ - ("model_name", native_field(item, "name")?), + ("model_name", model_reads::name_object(item)?), ( "origin", location(&self.runtime, &native_field(history, "old")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 840ef0b8ab..1aaa832716 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; @@ -42,7 +43,7 @@ pub(super) fn retention<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { old_name_retention( - &native_field(&native_field(item, "config")?, "values")?, + &native_field(&model_reads::config(item)?, "values")?, project, ) } @@ -70,7 +71,7 @@ pub(super) fn recorded_view<'py>( let created = native_field(history, "created")?; let mut reused = py.None().into_bound(py); for item in models { - if relation(&native_field(item, "destination")?)? + if relation(&model_reads::destination(item)?)? .call_method1("matches", (native_field(history, "old")?,))? .is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 16654086ab..a7eb581cc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -4,6 +4,8 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; @@ -24,7 +26,7 @@ fn schemas<'py>( let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { - let schema = native_field(&native_field(item, "destination")?, "schema")?; + let schema = native_field(&model_reads::destination(item)?, "schema")?; if !schema.is_none() { needed.add(schema)?; } @@ -43,7 +45,7 @@ fn schemas<'py>( if !include_project { for item in selected { let existing = native_field(&snapshot, "existing_relations")? - .call_method1("get", (native_field(item, "name")?,))?; + .call_method1("get", (model_reads::name_object(item)?,))?; if !existing.is_none() && host::entry(py, AdapterHost::NormalizeRelationType)? .call1((native_field(&existing, "relation_type")?,))? @@ -75,12 +77,12 @@ pub(crate) fn plan_old_name_views<'py>( let runtime = phase.runtime.clone(); let scope = phase.scope.clone(); let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let models: Vec> = native_field(&project, "models")? + let models: Vec> = project_reads::models(&project)? .try_iter()? .collect::>()?; let by_name = PyDict::new(py); for item in &models { - by_name.set_item(native_field(item, "name")?, item)?; + by_name.set_item(model_reads::name_object(item)?, item)?; } let mut selected: Vec> = Vec::new(); for key in native_field(&scope, "execution_order")?.try_iter()? { @@ -136,7 +138,7 @@ pub(crate) fn plan_old_name_views<'py>( } } for item in &selected { - if !covered.contains(native_field(item, "name")?)? + if !covered.contains(model_reads::name_object(item)?)? && let Some(planned) = planner.pending_for(item)? { entries.push(planned); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 09fabbd24d..516a9a1277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -22,7 +22,7 @@ pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { - let schema = native_field(model, "schema_entry")?; + let schema = model_reads::schema_entry(model)?; let mut columns: Vec<(String, Option)> = Vec::new(); if !schema.is_none() { for column in native_field(&schema, "columns")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 8ae3f96ed6..4639fdef9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -1,6 +1,8 @@ //! Resolve authored migration origins at the adapter and inspection boundaries. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -31,14 +33,11 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let item = item?; - if native_field(&item, "name")?.eq(name)? { + if model_reads::name_object(&item)?.eq(name)? { return Ok((native_field(&item, "destination")?, name.clone())); } } @@ -73,8 +72,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = native_field(&project, "effective_target_database")?; - for item in native_field(&project, "models")?.try_iter()? { - let database = native_field(&native_field(&item?, "destination")?, "database")?; + for item in project_reads::models(&project)?.try_iter()? { + let database = native_field(&model_reads::destination(&item?)?, "database")?; if !database.is_none() { return Ok(database); } @@ -85,13 +84,10 @@ pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for item in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { - let schema = native_field(&native_field(&item?, "destination")?, "schema")?; + let schema = native_field(&model_reads::destination(&item?)?, "schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -145,7 +141,7 @@ pub(super) fn resolve_origin<'py>( _ => super::column_objects::text(py, ""), }; let item = request.model.clone(); - let destination = native_field(&item, "destination")?; + let destination = model_reads::destination(&item)?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { parts.push( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 974ff73a14..ff17cb0f39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::cursor::intervals::partition; use crate::bindings::_helpers::cursor::scalars; use crate::bindings::_helpers::cursor::scalars::compare_values; @@ -194,7 +195,7 @@ fn runtime_producer_names( return Ok(names); } let mut loaded: HashSet = HashSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if !native_field(&native_field(&source, "source_entry")?, "loader")?.is_none() { loaded.insert(native_field(&source, "name")?.extract()?); @@ -341,7 +342,7 @@ fn collect_models<'py>( }; let runtime_names = runtime_producer_names(project, &producers)?; let mut models: Vec<_> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let name_value = model_reads::name_object(&model)?; let model_name: String = name_value.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index a63794be55..e64a922859 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{text, tuple}; @@ -192,7 +193,7 @@ pub(crate) fn semantic_source_columns<'py>( return Ok(columns.clone()); } let mut required: BTreeSet = BTreeSet::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let analysis_off: bool = matches!( *config_node_of(&model_values(&model)?, SQL_ANALYSIS_KEY)?, @@ -201,7 +202,7 @@ pub(crate) fn semantic_source_columns<'py>( if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis_off { continue; } - for reference in native_field(&model, "references")?.try_iter()? { + for reference in model_reads::references(&model)?.try_iter()? { let reference = reference?; if native_field(&reference, "ref_kind")?.eq("source")? { required.insert(native_field(&reference, "ref_name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index eb6a4f00d0..d208b15f31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; @@ -75,8 +76,13 @@ pub(crate) fn destination_nodes<'py>( project: &Bound<'py, PyAny>, ) -> PyResult>> { let mut nodes: Vec<_> = Vec::new(); - for field in ["models", "seeds", "functions"] { - for node in native_field(project, field)?.try_iter()? { + let collections: [Bound<'py, PyAny>; 3] = [ + project_reads::models(project)?, + project_reads::seeds(project)?, + project_reads::functions(project)?, + ]; + for collection in collections { + for node in collection.try_iter()? { nodes.push(node?); } } @@ -151,7 +157,7 @@ fn model_upstream_names( selected_names: &HashSet, ) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); - for reference in native_field(model, "references")?.try_iter()? { + for reference in model_reads::references(model)?.try_iter()? { let reference = reference?; let kind = native_field(&reference, "ref_kind")?; let name: String = native_field(&reference, "ref_name")?.extract()?; @@ -189,7 +195,7 @@ fn metadata_name_filter( let models = by_name(project, "models")?; let seeds = by_name(project, "seeds")?; let sources = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; sources.set_item(native_field(&entry, "name")?, entry)?; } @@ -271,7 +277,7 @@ fn prefetch_schemas<'py>( for schema in schemas { scopes.push(PyTuple::new(py, [database.clone(), text(py, schema)])?.into_any()); } - for source in native_field(&request.project, "sources")?.try_iter()? { + for source in project_reads::sources(&request.project)?.try_iter()? { let source = source?; let entry = native_field(&source, "source_entry")?; if (!relevant.is_none() && !relevant.contains(native_field(&source, "key")?)?) @@ -466,7 +472,7 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> .collect::>()?; let mut all_selected = true; let mut any_migration = false; - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !names.contains(&model_reads::model_name(&model)?) { all_selected = false; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b0f40a579d..e0c61fdbf8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -5,6 +5,7 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::native_host::host; @@ -27,7 +28,7 @@ use std::collections::BTreeSet; fn managed_names(project: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; if !native_field(&entry, "loader")?.is_none() { names.insert(native_field(&entry, "name")?.extract()?); @@ -87,7 +88,7 @@ fn selected_managed_refs( managed: &BTreeSet, ) -> PyResult> { let mut names: BTreeSet<_> = BTreeSet::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected.contains(model_reads::model_key(&model)?)? { continue; @@ -95,7 +96,7 @@ fn selected_managed_refs( names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if selected.contains(native_field(&function, "key")?)? { names.extend(sql_source_reads( @@ -104,7 +105,7 @@ fn selected_managed_refs( )?); } } - for audit in native_field(project, "audits")?.try_iter()? { + for audit in project_reads::audits(project)?.try_iter()? { let audit = audit?; let deps = native_field(&audit, "scope_deps")?; if deps.is_truthy()? { @@ -309,7 +310,7 @@ pub(crate) fn build_source_read_map<'py>( let target = resolve_target(project_config, local_config, target_name)?; let effective_vars = native_field(project, "effective_vars")?; let result = source_map.copy()?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let name: String = native_field(&native_field(&source, "source_entry")?, "name")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index aacf69c846..a84e66022e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; @@ -96,7 +97,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( let read_map = native_field(relations, "source_read_map")?; let listed = native_field(relations, "listed_source_names")?; let mut unlisted: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Vec)> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let name: String = native_field(&source, "name")?.extract()?; let Some(source_readers) = readers.get(&name).filter(|readers| !readers.is_empty()) else { From 713ace36f3fe18a5301842018b73d06ee90b15cd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:03:13 +0100 Subject: [PATCH 4/4] refactor(native): read remaining planner models and project collections natively Plan entries, phases, identity metadata, scenarios, selectors and the audit, seed, test, display and clone command plans read a model's name, destination, key, schema entry, config, references and enum columns, and iterate the project's models, sources, seeds, functions, audits and SQL tests, through the native view helpers. A named collection lookup serves the callers that pick the collection by name. --- .../_helpers/compiled_views/model_reads.rs | 9 +++++ .../_helpers/compiled_views/project_reads.rs | 40 ++++++++++++++++--- .../_helpers/planner_entries/clone_plans.rs | 28 ++++++------- .../_helpers/planner_entries/command_plans.rs | 11 ++--- .../_helpers/planner_entries/display_plan.rs | 3 +- .../_helpers/planner_entries/model_entries.rs | 3 +- .../_helpers/planner_entries/plan_output.rs | 7 ++-- .../_helpers/planner_entries/relations.rs | 9 +++-- .../_helpers/planner_identity/graph.rs | 15 +++++-- .../_helpers/planner_identity/identities.rs | 4 +- .../_helpers/planner_identity/metadata.rs | 10 ++--- .../_helpers/planner_phases/buildability.rs | 3 +- .../_helpers/planner_phases/full_refresh.rs | 3 +- .../_helpers/planner_phases/identities.rs | 5 ++- .../_helpers/planner_phases/objects.rs | 3 +- .../planner_phases/output_assembly.rs | 8 ++-- .../_helpers/planner_phases/retention.rs | 12 +++--- .../bindings/_helpers/planner_scenario/cli.rs | 3 +- .../_helpers/planner_scenario/entries.rs | 3 +- .../_helpers/planner_scenario/execution.rs | 9 +++-- .../_helpers/planner_scenario/graph.rs | 7 ++-- .../planner_selection/model_selectors.rs | 3 +- .../planner_warehouse/source_columns.rs | 3 +- .../planner_warehouse/source_deferral.rs | 2 +- .../planner_warehouse/source_tables.rs | 2 +- 25 files changed, 130 insertions(+), 75 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs index 4b5874bfa8..fb24ddc151 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/model_reads.rs @@ -314,3 +314,12 @@ pub(crate) fn deps<'py>(model: &Bound<'py, PyAny>) -> PyResult } native_field(model, "deps") } + +/// The model's enum columns by name. +pub(crate) fn enum_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + if let Some(view) = native_model(model)? { + return Ok(view.enum_columns(py)?.into_bound(py).into_any()); + } + native_field(model, "enum_columns") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs index dc81541f2f..a189750f87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/project_reads.rs @@ -6,6 +6,12 @@ use crate::bindings::_helpers::compile_driver::stage_work::{COMPILED_VIEW_FALLBA use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::records::fields::field as native_field; +const MODELS_FIELD: &str = "models"; +const SOURCES_FIELD: &str = "sources"; +const SEEDS_FIELD: &str = "seeds"; +const FUNCTIONS_FIELD: &str = "functions"; +const AUDITS_FIELD: &str = "audits"; + /// The native view behind a project object; any other object is a Python-built record, counted. #[inline] fn native_project<'a>(project: &'a Bound<'_, PyAny>) -> PyResult> { @@ -22,7 +28,7 @@ pub(crate) fn models<'py>(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult( + project: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + match field { + MODELS_FIELD => models(project), + SOURCES_FIELD => sources(project), + SEEDS_FIELD => seeds(project), + FUNCTIONS_FIELD => functions(project), + AUDITS_FIELD => audits(project), + _ => native_field(project, field), + } +} + +/// The project's compiled SQL tests. +pub(crate) fn sql_tests<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(view) = native_project(project)? { + return Ok(view.sql_tests(py).into_bound(py)); + } + native_field(project, "sql_tests") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 14c8f0fc90..8b54df5355 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::model_config::typed_config::{config_node_of, model_values}; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; @@ -121,13 +122,13 @@ fn resolved_source_entries<'py>( let source_type = enum_member(py, RecordId::CompiledResourceType, "SOURCE")?; let read = PySet::empty(py)?; let mut readers: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if recreated_view(&model, &selected)? { readers.push(model); } } - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if selected.contains(native_field(&function, "key")?)? { readers.push(function); @@ -157,14 +158,11 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = object( - py, - build_model_locations(&native_field(project, "models")?)?, - )?; - let seed_locations = object(py, build_seed_locations(&native_field(project, "seeds")?)?)?; + let model_locations = object(py, build_model_locations(&project_reads::models(project)?)?)?; + let seed_locations = object(py, build_seed_locations(&project_reads::seeds(project)?)?)?; let function_locations = object( py, - build_function_locations(&native_field(project, "functions")?)?, + build_function_locations(&project_reads::functions(project)?)?, )?; let source_map = PyDict::new(py); for source in native_field(source_project, "sources")?.try_iter()? { @@ -232,7 +230,7 @@ fn clone_model_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; @@ -240,7 +238,7 @@ fn clone_model_entries<'py>( let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; functions.set_item(native_field(&function, "name")?, function)?; } @@ -301,7 +299,7 @@ fn clone_source_entries<'py>( ) -> PyResult> { let py = project.py(); let entries = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; let entry = native_field(&source, "source_entry")?; if !selected.contains(native_field(&source, "key")?)? @@ -359,7 +357,7 @@ fn clone_function_entries<'py>( let py = project.py(); let selected = native_field(plan, "selected_keys")?; let entries = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if !selected.contains(native_field(&function, "key")?)? { continue; @@ -445,7 +443,7 @@ fn seed_entries<'py>( wanted: &Bound<'py, PyAny>, ) -> PyResult> { let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if wanted.contains(native_field(&seed, field)?)? { entries.push(seed_entry(&seed)?); @@ -460,7 +458,7 @@ fn origin_model_entries<'py>( ) -> PyResult> { let py = project.py(); let mut entries: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; @@ -487,7 +485,7 @@ fn origin_source_entries<'py>( let py = project.py(); let mut keys: Vec> = Vec::new(); let mut order: Vec> = Vec::new(); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; if selected_names.contains(native_field(&source, "name")?)? { keys.push(native_field(&source, "key")?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 6de0009d69..759b7fa9a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ @@ -159,7 +160,7 @@ pub(crate) fn build_audit_command_plan<'py>( ) -> PyResult> { let py = project.py(); let materializations = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; materializations.set_item( model_reads::name_object(&model)?, @@ -200,7 +201,7 @@ fn called_function_keys<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let functions: Bound<'py, PyDict> = PyDict::new(py); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; functions.set_item(native_field(&function, "key")?, function)?; } @@ -254,7 +255,7 @@ pub(crate) fn build_test_command_plan<'py>( )?; let (entries, warnings) = (selected.entries, selected.warnings); let mut case_names: BTreeSet = BTreeSet::new(); - for test in native_field(&project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(&project)?.try_iter()? { let test = test?; let case_name = native_field(&test, "case_name")?; if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { @@ -271,7 +272,7 @@ pub(crate) fn build_test_command_plan<'py>( } let function_keys = called_function_keys(&project, function_keys)?; let mut functions: Vec> = Vec::new(); - for function in native_field(&project, "functions")?.try_iter()? { + for function in project_reads::functions(&project)?.try_iter()? { let function = function?; if !function_keys.contains(native_field(&function, "key")?)? { continue; @@ -331,7 +332,7 @@ pub(crate) fn build_seed_command_plan<'py>( }; let selected_keys = native_field(scope, "selected_keys")?; let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys.contains(native_field(&seed, "key")?)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index caf8713de1..1d67e3d651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,6 +1,7 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_text_object, model_values}; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, @@ -138,7 +139,7 @@ pub(crate) fn build_display_only_sqlbuild_plan<'py>( .collect::>>()?, )?; let mut entries: Vec> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if selected.contains(model_reads::name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index ae9d789bc0..ddba156a81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; @@ -426,7 +427,7 @@ pub(crate) fn plan_entries<'py>( runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); - for function in native_field(&project, "functions")?.try_iter()? { + for function in project_reads::functions(&project)?.try_iter()? { let function = function?; functions_by_name.set_item(native_field(&function, "name")?, function)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index afbf586a68..8aebad4969 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ @@ -104,7 +105,7 @@ fn seed_entries<'py>( let metadata_jsons = native_field(extras, "seed_metadata_jsons")?; let plan_reasons = native_field(extras, "seed_plan_reasons")?; let mut entries: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { let seed = seed?; if !selected_keys.contains(native_field(&seed, "key")?)? { continue; @@ -404,7 +405,7 @@ fn function_entries<'py>( let functions = native_field(changes, "functions")?; let previous = native_field(&native_field(snapshot, "fingerprints")?, "functions")?; let mut entries: Vec> = Vec::new(); - for function in native_field(project, "functions")?.try_iter()? { + for function in project_reads::functions(project)?.try_iter()? { let function = function?; if !selected_keys.contains(native_field(&function, "key")?)? { continue; @@ -464,7 +465,7 @@ pub(super) fn selected_test_entries<'py>( ) -> PyResult> { let py = project.py(); let mut selected: Vec> = Vec::new(); - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test = test?; if !test_selected(&test, selected_keys, selection)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index b04b65bc6c..1e4797cbac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; @@ -31,7 +32,7 @@ fn selected_audit_relation_keys<'py>( ) -> PyResult> { let py = project.py(); let keys = PySet::empty(py)?; - for audit in native_field(project, "audits")?.try_iter()? { + for audit in project_reads::audits(project)?.try_iter()? { let audit = audit?; if audit_is_selected(&audit, selected_keys)? { for dependency in native_field(&audit, "scope_deps")?.try_iter()? { @@ -99,12 +100,12 @@ pub(crate) fn planner_relations_context<'py>( let selected_keys = native_field(&scope, "selected_keys")?; let mut model_locations = object( py, - build_model_locations(&native_field(&project, "models")?)?, + build_model_locations(&project_reads::models(&project)?)?, )?; - let mut seed_locations = object(py, build_seed_locations(&native_field(&project, "seeds")?)?)?; + let mut seed_locations = object(py, build_seed_locations(&project_reads::seeds(&project)?)?)?; let function_locations = object( py, - build_function_locations(&native_field(&project, "functions")?)?, + build_function_locations(&project_reads::functions(&project)?)?, )?; let deferred_locations = native_field(&deferral, "deferred_locations")?; if !deferred_locations.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 3130f2b735..40ac2bf9cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -20,6 +20,7 @@ use sqlbuild_analysis::planner_identity::main::topological_order::topological_or use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_identity::hashing::graph_key; @@ -89,8 +90,14 @@ pub(crate) fn build_execution_upstream_deps<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let upstream: Bound<'py, PyDict> = PyDict::new(py); - for collection in ["models", "sources", "seeds", "functions"] { - for resource in native_field(project, collection)?.try_iter()? { + let collections: [Bound<'py, PyAny>; 4] = [ + project_reads::models(project)?, + project_reads::sources(project)?, + project_reads::seeds(project)?, + project_reads::functions(project)?, + ]; + for collection in collections { + for resource in collection.try_iter()? { let resource: Bound<'py, PyAny> = resource?; upstream.set_item( native_field(&resource, "key")?, @@ -103,7 +110,7 @@ pub(crate) fn build_execution_upstream_deps<'py>( )?; } } - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; let test_key: Bound<'py, PyAny> = native_field(&test, "key")?; let scope_deps: Bound<'py, PyAny> = native_field(&test, "scope_deps")?; @@ -172,7 +179,7 @@ pub(crate) fn build_execution_edge_origins<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let origins: Bound<'py, PyDict> = PyDict::new(py); - for test in native_field(project, "sql_tests")?.try_iter()? { + for test in project_reads::sql_tests(project)?.try_iter()? { let test: Bound<'py, PyAny> = test?; let test_name: String = native_field(&test, "name")?.extract()?; for target_key in native_field(&test, "scope_deps")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 0f59644b3a..0e56c7adad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -180,7 +180,7 @@ pub(crate) fn direct_model_version_identities<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name), IdentityNode { - upstream_keys: graph_keys(&native_field(&model, "deps")?)?, + upstream_keys: graph_keys(&model_reads::deps(&model)?)?, local_hash: Some(local_hash), }, ); @@ -291,7 +291,7 @@ pub(crate) fn with_honest_model_write_hashes<'py>( graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { - upstream_keys: graph_keys(&native_field(&model, "deps")?)?, + upstream_keys: graph_keys(&model_reads::deps(&model)?)?, local_hash: mapping_get(&local_hashes, &name)?.extract()?, }, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 10e6618d2b..3be7c2a31b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -90,7 +90,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let py: Python<'py> = model.py(); let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { - for upstream_key in native_field(model, "deps")?.try_iter()? { + for upstream_key in model_reads::deps(model)?.try_iter()? { let upstream_key: Bound<'py, PyAny> = upstream_key?; let kind: String = native_field(&upstream_key, "resource_type")? .str()? @@ -105,7 +105,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( } } } - let config_values: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")?; + let config_values: Bound<'py, PyAny> = native_field(&model_reads::config(model)?, "values")?; let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( @@ -170,7 +170,7 @@ fn contract_output_signature<'py>( config_values: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = model.py(); - let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'py, PyAny> = model_reads::schema_entry(model)?; if schema_entry.is_none() { return Ok(None); } @@ -184,7 +184,7 @@ fn contract_output_signature<'py>( if !enforced && native_field(&schema_entry, "model_schema")?.is_none() { return Ok(None); } - let enum_columns: Bound<'py, PyAny> = native_field(model, "enum_columns")?; + let enum_columns: Bound<'py, PyAny> = model_reads::enum_columns(model)?; let rendered: Bound<'py, PyList> = PyList::empty(py); for column in columns.try_iter()? { let column: Bound<'py, PyAny> = column?; @@ -247,7 +247,7 @@ fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult) -> PyResult { let py: Python<'_> = model.py(); - let schema_entry: Bound<'_, PyAny> = native_field(model, "schema_entry")?; + let schema_entry: Bound<'_, PyAny> = model_reads::schema_entry(model)?; let payload: Bound<'_, PyDict> = PyDict::new(py); payload.set_item( "contract", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 7f5241fcb3..fb58fa50d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,6 +1,7 @@ //! Selected-scope buildability validation before change planning. use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_identity::graph::build_execution_edge_origins; use crate::bindings::_helpers::planner_identity::graph::check_buildability; use crate::bindings::_helpers::planner_identity::graph::missing_upstream_message; @@ -28,7 +29,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( let project = runtime_field(runtime, |runtime| &runtime.project)?; let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let mut external_seeds: Vec<_> = Vec::new(); - for seed in native_field(&project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(&project)?.try_iter()? { let seed = seed?; if native_field(&seed, "external")?.is_truthy()? { external_seeds.push(native_field(&seed, "key")?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index fc61483b4a..cef30a198f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,6 +1,7 @@ //! Per-model full-refresh resolution and full-rebuild refusals. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; @@ -39,7 +40,7 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( cli_full_refresh: bool, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { names.push(model_reads::name_object(&model)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index a869e16d2a..2125419581 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,6 +1,7 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; @@ -49,10 +50,10 @@ fn version_identities<'py>( ) -> PyResult> { identity_work(project.py())?.identity_builds += 1; direct_model_version_identities( - (&native_field(project, "functions")?, &scope), + (&project_reads::functions(project)?, &scope), native_field(project, "sql_analysis_dialect")?.extract()?, ( - &native_field(project, "seeds")?, + &project_reads::seeds(project)?, &project.py().None().into_bound(project.py()), &native_field(project, "hook_functions")?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 4376b8d325..acac698320 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -3,6 +3,7 @@ use sqlbuild_host::entries::models::HostId; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_text, model_values}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::error_class; @@ -150,7 +151,7 @@ pub(crate) fn by_name<'py>( field: &str, ) -> PyResult> { let nodes = pyo3::types::PyDict::new(project.py()); - for node in native_field(project, field)?.try_iter()? { + for node in project_reads::collection(project, field)?.try_iter()? { let node = node?; pyo3::types::PyDictMethods::set_item(&nodes, native_field(&node, "name")?, node)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 7259f08623..0c26149e6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; @@ -175,11 +176,8 @@ pub(crate) fn add_storage_policies<'py>( let mut entries = output.field("model_entries")?; if native_field(policies, "record_migration_fingerprints")?.is_truthy()? { let models = PyDict::new(py); - for model in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for model in project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)? + .try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index f48921bebd..680e3d1de2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -4,6 +4,7 @@ use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::model_config::typed_config::{config_value, typed_values}; use crate::bindings::_helpers::native_host::host; @@ -62,11 +63,8 @@ fn selected_models<'py>( .into_iter() .collect(); let mut models: Vec<_> = Vec::new(); - for model in native_field( - &runtime_field(runtime, |runtime| &runtime.project)?, - "models", - )? - .try_iter()? + for model in + project_reads::models(&runtime_field(runtime, |runtime| &runtime.project)?)?.try_iter()? { let model = model?; if names.contains(&model_reads::model_name(&model)?) { @@ -94,7 +92,7 @@ pub(crate) fn plan_table_types<'py>( let mut declared: Vec<_> = Vec::new(); for model in &models { if native_field( - &native_field(&native_field(model, "config")?, "table_type")?, + &native_field(&model_reads::config(model)?, "table_type")?, "declared", )? .is_truthy()? @@ -595,7 +593,7 @@ pub(crate) fn plan_retention<'py>( let py = runtime.py(); let mut models: Vec<_> = Vec::new(); for model in selected_models(runtime, scope)? { - let retention = native_field(&native_field(&model, "config")?, "time_travel_retention")?; + let retention = native_field(&model_reads::config(&model)?, "time_travel_retention")?; if !native_field(&retention, "unmanaged")?.is_truthy()? && !native_field(&retention, "desired_days")?.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 39b573ed81..15e418f408 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; @@ -138,7 +139,7 @@ fn namespace<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let models = PyDict::new(py); - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index e506f21877..561b359710 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; @@ -23,7 +24,7 @@ pub(super) fn seed_entries<'py>( relations: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let seeds = by_attribute(&native_field(project, "seeds")?, "name")?; + let seeds = by_attribute(&project_reads::seeds(project)?, "name")?; let fixtures = native_field(graph, "seed_fixture_names")?; let mut result: Vec> = Vec::new(); for name in native_field(graph, "seed_names")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index e209fbbc4c..7328fe60e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::boundary::python_calls::keywords; use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::compiled_views::records::compiled_model_record; use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; @@ -89,9 +90,9 @@ pub(crate) fn build_scenario_execution_plan<'py>( } else { analysis.is_truthy()? }; - let models = by_attribute(&native_field(&project, "models")?, "name")?; - let functions = by_attribute(&native_field(&project, "functions")?, "key")?; - let locations = build_function_locations(&native_field(&project, "functions")?)?.into_any(); + let models = by_attribute(&project_reads::models(&project)?, "name")?; + let functions = by_attribute(&project_reads::functions(&project)?, "key")?; + let locations = build_function_locations(&project_reads::functions(&project)?)?.into_any(); let upstream = build_execution_upstream_deps(&project)?.into_any(); let ordered = object(py, topologically_order_keys(&upstream, None)?)?; let mut model_keys: Vec> = Vec::new(); @@ -267,7 +268,7 @@ fn model_context<'py>( ModelPlanContext { model_locations: native_field(&context.relations, "model_locations")?.unbind(), models_by_name: models.clone().into_any().unbind(), - functions_by_name: by_attribute(&native_field(project, "functions")?, "name")? + functions_by_name: by_attribute(&project_reads::functions(project)?, "name")? .into_any() .unbind(), seed_locations: native_field(&context.relations, "seed_locations")?.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 525142eac8..29039b9dd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,6 +1,7 @@ //! Convert project views into pure scenario dependency facts. use crate::bindings::_helpers::compiled_views::model_reads; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::values::string; @@ -33,7 +34,7 @@ pub(crate) fn plan_scenario_graph<'py>( dbt_refs: native_field(scenario, "dbt_ref_fixture_names")?.extract()?, ..ScenarioGraphInput::default() }; - for model in native_field(project, "models")?.try_iter()? { + for model in project_reads::models(project)?.try_iter()? { let model = model?; let deps = native_field(&model, "deps")? .try_iter()? @@ -43,12 +44,12 @@ pub(crate) fn plan_scenario_graph<'py>( .models .insert(model_reads::name_object(&model)?.extract()?, deps); } - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { input .project_sources .insert(native_field(&source?, "name")?.extract()?); } - for seed in native_field(project, "seeds")?.try_iter()? { + for seed in project_reads::seeds(project)?.try_iter()? { input .project_seeds .insert(native_field(&seed?, "name")?.extract()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs index 34c0701681..78848545cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::planner_selection::models::{ }; use crate::bindings::_helpers::boundary::python_calls::imported; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::model_config::typed_config::{config_value, model_values}; use crate::bindings::_helpers::planner_selection::carrier::Carrier; use crate::bindings::_helpers::records::construct::new_record_with; @@ -88,7 +89,7 @@ pub(crate) fn resolve_sqlbuild_model_selector_names<'py>( term: &Bound<'py, PyString>, ) -> PyResult> { let py: Python<'py> = project.py(); - let models: Vec = native_field(project, "models")? + let models: Vec = project_reads::models(project)? .try_iter()? .map(|model| selectable_model(&model?)) .collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs index 7c63930fa3..6f0bcca6b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_columns.rs @@ -1,6 +1,7 @@ //! Warehouse source relation listing and column gathering for planning. use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::compiled_views::project_reads; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{frozenset, text, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ @@ -19,7 +20,7 @@ fn entries_of<'py>( ) -> PyResult>> { match source_entries.filter(|entries| !entries.is_none()) { Some(entries) => entries.try_iter()?.collect(), - None => native_field(project, "sources")? + None => project_reads::sources(project)? .try_iter()? .map(|source| native_field(&source?, "source_entry")) .collect(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index e0c61fdbf8..502d7bf9e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -345,7 +345,7 @@ pub(crate) fn with_declared_source_reads<'py>( return Ok(source_map.clone()); } let result = source_map.copy()?; - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let entry = native_field(&source?, "source_entry")?; let name: String = native_field(&entry, "name")?.extract()?; if missing.contains(&name) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index a84e66022e..d41bb47bc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -174,7 +174,7 @@ pub(crate) fn check_selected_source_tables_exist<'py>( } let message = missing_source_tables_message(&missing); let sources = PyDict::new(py); - for source in native_field(project, "sources")?.try_iter()? { + for source in project_reads::sources(project)?.try_iter()? { let source = source?; sources.set_item(native_field(&source, "name")?, source)?; }