From 680997fd0bcbf33ab8944047f8c01e6ed24fe88f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:34:36 +0100 Subject: [PATCH 1/4] refactor(planner): hold planner scope, identity, relation and entry contexts in Rust PlannerScopeResolution, PlannerIdentityContext, PlannerRelationsContext, ModelPlanContext and PlanEntryBuildInputs are frozen native classes with typed fields instead of generic native records. Plan entry construction, SQL resolution and cursor bound computation read their fields directly. Constructors take the same keywords; the classes no longer support dataclasses.replace. --- .../_helpers/planner_entries/clone_plans.rs | 37 ++-- .../_helpers/planner_entries/contexts.rs | 171 ++++++++++++++++++ .../planner_entries/execution_plan.rs | 19 +- .../bindings/_helpers/planner_entries/mod.rs | 1 + .../_helpers/planner_entries/model_entries.rs | 96 +++++----- .../_helpers/planner_entries/models.rs | 52 +++++- .../_helpers/planner_entries/registration.rs | 5 +- .../_helpers/planner_entries/relations.rs | 35 ++-- .../_helpers/planner_phases/buildability.rs | 4 +- .../_helpers/planner_phases/contexts.rs | 49 +++++ .../planner_phases/execution_changes.rs | 77 ++++---- .../_helpers/planner_phases/identities.rs | 41 +++-- .../bindings/_helpers/planner_phases/mod.rs | 1 + .../_helpers/planner_phases/models.rs | 17 ++ .../planner_phases/output_assembly.rs | 23 ++- .../_helpers/planner_phases/registration.rs | 2 + .../_helpers/planner_phases/scopes.rs | 19 +- .../planner_phases/warehouse_state.rs | 9 +- .../_helpers/planner_records/constants.rs | 80 -------- .../planner_resolution/cursor_model.rs | 26 ++- .../planner_resolution/cursor_ranges.rs | 12 +- .../planner_resolution/orchestration.rs | 14 +- .../_helpers/planner_scenario/execution.rs | 52 +++--- .../src/bindings/_helpers/records/ids.rs | 5 - src/sqlbuild/_native.pyi | 19 +- .../planner/_helpers/resolve/helpers.py | 16 +- .../planner/_helpers/resolve/test_hooks.py | 5 +- 27 files changed, 578 insertions(+), 309 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 99d1c06bd9..d5d677a858 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; @@ -190,26 +191,24 @@ fn clone_model_entries<'py>( let function = function?; functions.set_item(function.getattr("name")?, function)?; } - let context = native( + let context = Bound::new( py, - RecordId::ModelPlanContext, - &[ - ("model_locations", resolution.model_locations.clone()), - ("models_by_name", PyDict::new(py).into_any()), - ("functions_by_name", functions.into_any()), - ("seed_locations", resolution.seed_locations.clone()), - ("function_locations", resolution.function_locations.clone()), - ("source_map", resolution.source_map.clone()), - ( - "source_warehouse_columns", - resolution.source_warehouse_columns.clone(), - ), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ], - )?; + ModelPlanContext { + model_locations: resolution.model_locations.clone().unbind(), + models_by_name: PyDict::new(py).into_any().unbind(), + functions_by_name: functions.into_any().unbind(), + seed_locations: resolution.seed_locations.clone().unbind(), + function_locations: resolution.function_locations.clone().unbind(), + source_map: resolution.source_map.clone().unbind(), + source_warehouse_columns: resolution.source_warehouse_columns.clone().unbind(), + star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), + future_cursor_config: py.None(), + start_cursor_config: py.None(), + invocation_time: py.None(), + }, + )? + .into_any(); let sql = resolve_model_sql( &Resolution::of_context(adapter.clone(), &context, py.None().into_bound(py))?, &context, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs new file mode 100644 index 0000000000..44111a26d4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs @@ -0,0 +1,171 @@ +//! Relation, model-plan and entry-build contexts: built from keyword arguments and read by plan entry construction. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::{PyDict, PyFrozenSet}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::{ + ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, +}; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; + +/// The argument `name`, or `default()` when it was not passed. +fn value_or<'py>( + args: &KeywordArguments<'py>, + name: &str, + default: impl FnOnce() -> PyResult>, +) -> PyResult> { + Ok(if args.passed(name)? { + args.value(name)? + } else { + default()? + } + .unbind()) +} + +fn empty_frozenset(py: Python<'_>) -> PyResult> { + Ok(PyFrozenSet::empty(py)?.into_any()) +} + +#[pymethods] +impl PlannerRelationsContext { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerRelationsContext", + kwargs, + ( + &[ + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_read_map", + "python_source_read_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &["listed_source_names"], + ), + )?; + Ok(Self { + model_locations: args.value("model_locations")?.unbind(), + seed_locations: args.value("seed_locations")?.unbind(), + function_locations: args.value("function_locations")?.unbind(), + source_map: args.value("source_map")?.unbind(), + source_read_map: args.value("source_read_map")?.unbind(), + python_source_read_map: args.value("python_source_read_map")?.unbind(), + source_warehouse_columns: args.value("source_warehouse_columns")?.unbind(), + star_exclude_keyword: args.value("star_exclude_keyword")?.unbind(), + listed_source_names: value_or(&args, "listed_source_names", || empty_frozenset(py))?, + }) + } +} + +#[pymethods] +impl ModelPlanContext { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "ModelPlanContext", + kwargs, + ( + &[ + "model_locations", + "models_by_name", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &[ + "functions_by_name", + "runtime_cursor_producer_names", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + ], + ), + )?; + Ok(Self { + model_locations: args.value("model_locations")?.unbind(), + models_by_name: args.value("models_by_name")?.unbind(), + seed_locations: args.value("seed_locations")?.unbind(), + function_locations: args.value("function_locations")?.unbind(), + source_map: args.value("source_map")?.unbind(), + source_warehouse_columns: args.value("source_warehouse_columns")?.unbind(), + star_exclude_keyword: args.value("star_exclude_keyword")?.unbind(), + functions_by_name: value_or(&args, "functions_by_name", || { + Ok(PyDict::new(py).into_any()) + })?, + runtime_cursor_producer_names: value_or( + &args, + "runtime_cursor_producer_names", + || empty_frozenset(py), + )?, + future_cursor_config: args.value("future_cursor_config")?.unbind(), + start_cursor_config: args.value("start_cursor_config")?.unbind(), + invocation_time: args.value("invocation_time")?.unbind(), + }) + } +} + +#[pymethods] +impl PlanEntryBuildInputs { + #[new] + #[pyo3(signature = (**kwargs))] + pub(crate) fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlanEntryBuildInputs", + kwargs, + ( + &[], + &[ + "source_freshness_blocked_model_names", + "external_blocked_model_names", + "start_cursor_override", + "end_cursor_override", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + "max_microbatches", + "max_microbatches_is_override", + "microbatch_limit_action", + ], + ), + )?; + Ok(Self { + source_freshness_blocked_model_names: value_or( + &args, + "source_freshness_blocked_model_names", + || empty_frozenset(py), + )?, + external_blocked_model_names: value_or(&args, "external_blocked_model_names", || { + empty_frozenset(py) + })?, + start_cursor_override: args.value("start_cursor_override")?.unbind(), + end_cursor_override: args.value("end_cursor_override")?.unbind(), + future_cursor_config: args.value("future_cursor_config")?.unbind(), + start_cursor_config: args.value("start_cursor_config")?.unbind(), + invocation_time: args.value("invocation_time")?.unbind(), + max_microbatches: args.value("max_microbatches")?.unbind(), + max_microbatches_is_override: args.flag("max_microbatches_is_override")?, + microbatch_limit_action: value_or(&args, "microbatch_limit_action", || { + default_microbatch_limit_action(py) + })?, + }) + } +} + +/// The microbatch limit action used when a project configures none. +pub(crate) fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { + registry::class(py, RecordId::MicrobatchLimitAction)?.getattr("ERROR") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 74cbae755f..068538d9ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -13,13 +13,16 @@ use crate::bindings::_helpers::planner_entries::test_planning::{ }; use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::execution_changes::{ build_planner_entry_results, reconcile_execution_changes, }; use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, }; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; @@ -46,7 +49,7 @@ fn scopes_and_cursor_overrides<'py>( &args.value("selection")?, &args.value("policies")?, )?; - let selected = scopes.getattr("selected_scope")?; + let selected = class_field(&scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let models_by_name = selected.getattr("models_by_name")?; let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; let mut models: Vec> = Vec::new(); @@ -73,9 +76,15 @@ fn change_results<'py>( policies: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); - let versions = identities.getattr("version_identities")?; + let versions = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; let changes = detect_changes( - (project, &scopes.getattr("inspection_scope")?, snapshot), + ( + project, + &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + snapshot, + ), overrides.getattr("full_refresh")?.is_truthy()?, ( versions.getattr("model_version_hashes")?, @@ -123,7 +132,7 @@ fn planned_output<'py>( overrides, &policies, )?; - let inspection_scope = scopes.getattr("inspection_scope")?; + let inspection_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; let source_freshness = build_planner_source_freshness_result( (&project, &inspection_scope), @@ -212,7 +221,7 @@ fn build_execution_plan<'py>( )? .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; - let inspection = scopes.getattr("inspection_scope")?; + let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; let test_planning = start_planning( py, PlanningRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 519d1ac918..5e31964014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -2,6 +2,7 @@ mod clone_plans; mod command_plans; +pub(crate) mod contexts; mod display_plan; mod execution_plan; pub(crate) mod model_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 5b9898a4b9..1355be400e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -3,11 +3,14 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; -use crate::bindings::_helpers::planner_entries::models::PlanEntriesRequest; +use crate::bindings::_helpers::planner_entries::models::{ + ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, +}; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, }; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; @@ -36,7 +39,7 @@ use crate::bindings::_helpers::planner_strategy::strategy::{ use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -56,18 +59,21 @@ impl<'py> ContextReads<'py> { adapter: Bound<'py, PyAny>, external_resolver: Bound<'py, PyAny>, ) -> PyResult { + let py = context.py(); + let fields = context.cast::()?.get(); + let field = |value: &Py| value.bind(py).clone(); Ok(Self { lookups: CursorInputLookups { - models_by_name: context.getattr("models_by_name")?, - functions_by_name: context.getattr("functions_by_name")?, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + models_by_name: field(&fields.models_by_name), + functions_by_name: field(&fields.functions_by_name), + source_map: field(&fields.source_map), + source_warehouse_columns: field(&fields.source_warehouse_columns), + model_locations: field(&fields.model_locations), + seed_locations: field(&fields.seed_locations), + runtime_cursor_producer_names: field(&fields.runtime_cursor_producer_names), }, resolution: Resolution::of_context(adapter, &context, external_resolver)?, - invocation_time: context.getattr("invocation_time")?, + invocation_time: field(&fields.invocation_time), context, }) } @@ -227,7 +233,12 @@ fn check_binding_diagnostics<'py>( NativeAttr::SourceBindingDiagnostics, &[ ("project", project.clone()), - ("columns", relations.getattr("source_warehouse_columns")?), + ( + "columns", + class_field(relations, |context: &PlannerRelationsContext| { + &context.source_warehouse_columns + })?, + ), ( "profile", adapter.call_method0("expression_inference_profile")?, @@ -385,22 +396,24 @@ pub(crate) fn plan_entries<'py>( } = request; let py = project.py(); let inputs = if inputs.is_none() { - native(py, RecordId::PlanEntryBuildInputs, &[])? + Bound::new(py, PlanEntryBuildInputs::new(py, None)?)?.into_any() } else { inputs }; + let build = inputs.cast::()?.get(); + let relations_context = relations.cast::()?.get(); let selected_keys = scope.getattr("selected_keys")?; check_binding_diagnostics(&project, &adapter, &relations, &selected_keys)?; - let freshness_blocked = inputs.getattr("source_freshness_blocked_model_names")?; - let external_blocked = inputs.getattr("external_blocked_model_names")?; - let start_override = inputs.getattr("start_cursor_override")?; - let end_override = inputs.getattr("end_cursor_override")?; + let freshness_blocked = build.source_freshness_blocked_model_names.bind(py).clone(); + let external_blocked = build.external_blocked_model_names.bind(py).clone(); + let start_override = build.start_cursor_override.bind(py).clone(); + let end_override = build.end_cursor_override.bind(py).clone(); let producers = native( py, NativeAttr::RuntimeCursorProducerNames, &[ ("scope", scope.clone()), - ("source_map", relations.getattr("source_map")?), + ("source_map", relations_context.source_map.bind(py).clone()), ], )?; let functions_by_name = PyDict::new(py); @@ -410,39 +423,24 @@ pub(crate) fn plan_entries<'py>( } let settings = project.getattr("settings")?; let models_by_name = scope.getattr("models_by_name")?; - let context = native( + let context = Bound::new( py, - RecordId::ModelPlanContext, - &[ - ("model_locations", relations.getattr("model_locations")?), - ("models_by_name", models_by_name.clone()), - ("functions_by_name", functions_by_name.into_any()), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", relations.getattr("source_read_map")?), - ( - "source_warehouse_columns", - relations.getattr("source_warehouse_columns")?, - ), - ( - "star_exclude_keyword", - relations.getattr("star_exclude_keyword")?, - ), - ("runtime_cursor_producer_names", producers), - ( - "future_cursor_config", - inputs.getattr("future_cursor_config")?, - ), - ( - "start_cursor_config", - inputs.getattr("start_cursor_config")?, - ), - ("invocation_time", inputs.getattr("invocation_time")?), - ], - )?; + ModelPlanContext { + model_locations: relations_context.model_locations.clone_ref(py), + models_by_name: models_by_name.clone().unbind(), + functions_by_name: functions_by_name.into_any().unbind(), + seed_locations: relations_context.seed_locations.clone_ref(py), + function_locations: relations_context.function_locations.clone_ref(py), + source_map: relations_context.source_read_map.clone_ref(py), + source_warehouse_columns: relations_context.source_warehouse_columns.clone_ref(py), + star_exclude_keyword: relations_context.star_exclude_keyword.clone_ref(py), + runtime_cursor_producer_names: producers.unbind(), + future_cursor_config: build.future_cursor_config.clone_ref(py), + start_cursor_config: build.start_cursor_config.clone_ref(py), + invocation_time: build.invocation_time.clone_ref(py), + }, + )? + .into_any(); let reads = ContextReads::of( context, adapter.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index 0cc640c9cb..6692769335 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -1,6 +1,7 @@ //! Typed requests of plan entry construction. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; /// What resolving relation locations and source metadata for one scope reads. #[derive(Debug)] @@ -44,3 +45,52 @@ pub(crate) struct PlanOutputInputs<'py> { pub(crate) reload_sources: Bound<'py, PyAny>, pub(crate) extras: Bound<'py, PyAny>, } + +/// Resolved relation and source inputs for plan entry construction. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerRelationsContext { + pub(crate) model_locations: Py, + pub(crate) seed_locations: Py, + pub(crate) function_locations: Py, + pub(crate) source_map: Py, + pub(crate) source_read_map: Py, + pub(crate) python_source_read_map: Py, + pub(crate) source_warehouse_columns: Py, + pub(crate) star_exclude_keyword: Py, + pub(crate) listed_source_names: Py, +} + +/// Relation lookups and source columns for building one model plan entry. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct ModelPlanContext { + pub(crate) model_locations: Py, + pub(crate) models_by_name: Py, + pub(crate) seed_locations: Py, + pub(crate) function_locations: Py, + pub(crate) source_map: Py, + pub(crate) source_warehouse_columns: Py, + pub(crate) star_exclude_keyword: Py, + pub(crate) functions_by_name: Py, + pub(crate) runtime_cursor_producer_names: Py, + pub(crate) future_cursor_config: Py, + pub(crate) start_cursor_config: Py, + pub(crate) invocation_time: Py, +} + +/// Blocked models and cursor overrides for plan entry building. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlanEntryBuildInputs { + pub(crate) source_freshness_blocked_model_names: Py, + pub(crate) external_blocked_model_names: Py, + pub(crate) start_cursor_override: Py, + pub(crate) end_cursor_override: Py, + pub(crate) future_cursor_config: Py, + pub(crate) start_cursor_config: Py, + pub(crate) invocation_time: Py, + pub(crate) max_microbatches: Py, + pub(crate) max_microbatches_is_override: bool, + pub(crate) microbatch_limit_action: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index 15f31ed296..ed45e8827f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -1,8 +1,11 @@ //! Register plan entry and plan output assembly entry points. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; super::clone_plans::register(module)?; super::command_plans::register(module)?; super::display_plan::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 7a2e29c8e4..94c0e1bf6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,7 +1,9 @@ //! Relation locations and source metadata for plan entry construction. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; +use crate::bindings::_helpers::planner_entries::models::{ + PlannerRelationsContext, RelationsRequest, +}; use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -192,24 +194,21 @@ pub(crate) fn planner_relations_context<'py>( ("columns", columns), ], )?; - planner_model( + Ok(Bound::new( py, - RecordId::PlannerRelationsContext, - &[ - ("model_locations", model_locations), - ("seed_locations", seed_locations), - ("function_locations", function_locations), - ("source_map", source_map), - ("source_read_map", source_read_map), - ("python_source_read_map", python_source_read_map), - ("source_warehouse_columns", semantic_columns), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ("listed_source_names", listed_source_names), - ], - ) + PlannerRelationsContext { + model_locations: model_locations.unbind(), + seed_locations: seed_locations.unbind(), + function_locations: function_locations.unbind(), + source_map: source_map.unbind(), + source_read_map: source_read_map.unbind(), + python_source_read_map: python_source_read_map.unbind(), + source_warehouse_columns: semantic_columns.unbind(), + star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + listed_source_names: listed_source_names.unbind(), + }, + )? + .into_any()) } /// Resolve relation locations and source metadata for plan entry construction. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 4372fa93b3..b032188b7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,6 +1,8 @@ //! Selected-scope buildability validation before change planning. use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; @@ -19,7 +21,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( ) -> PyResult<()> { let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; - let selected_scope = scopes.getattr("selected_scope")?; + let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let mut external_seeds: Vec<_> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { let seed = seed?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs new file mode 100644 index 0000000000..7525a1875b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -0,0 +1,49 @@ +//! Planner scope and identity contexts: built from keywords and read by planner phases. + +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::pymethods; +use pyo3::{PyClass, pyclass::boolean_struct::True}; + +use crate::bindings::_helpers::planner_phases::models::{ + PlannerIdentityContext, PlannerScopeResolution, +}; + +#[pymethods] +impl PlannerScopeResolution { + #[new] + #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] + fn new( + selected_scope: Py, + stale_warning_scope: Py, + inspection_scope: Py, + ) -> Self { + Self { + selected_scope, + stale_warning_scope, + inspection_scope, + } + } +} + +#[pymethods] +impl PlannerIdentityContext { + #[new] + #[pyo3(signature = (*, version_identities, stale_warning_identities))] + fn new(version_identities: Py, stale_warning_identities: Py) -> Self { + Self { + version_identities, + stale_warning_identities, + } + } +} + +/// One field of a frozen native planner class passed as a Python object. +pub(crate) fn class_field<'py, T>( + value: &Bound<'py, PyAny>, + field: fn(&T) -> &Py, +) -> PyResult> +where + T: PyClass + Sync, +{ + Ok(field(value.cast::()?.get()).bind(value.py()).clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 460fcb822d..c4f6f2780e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,14 +1,18 @@ //! Write-hash reconciliation and plan entry construction. +use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; -use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, RelationsRequest}; +use crate::bindings::_helpers::planner_entries::models::{ + PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest, +}; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::IntoPyObject; +use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; @@ -29,7 +33,9 @@ pub(crate) fn reconcile_execution_changes<'py>( ("changes", changes.clone()), ( "version_identities", - identities.getattr("version_identities")?, + class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?, ), ], )?; @@ -99,52 +105,37 @@ pub(crate) fn build_planner_entry_results<'py>( max_override.clone() }; let limit_action = if config.is_none() { - native(py, RecordId::PlanEntryBuildInputs, &[])?.getattr("microbatch_limit_action")? + default_microbatch_limit_action(py)? } else { config .getattr("microbatches")? .getattr("limits")? .getattr("action")? }; - let build_inputs = native( + let build_inputs = Bound::new( py, - RecordId::PlanEntryBuildInputs, - &[ - ("source_freshness_blocked_model_names", blocked), - ( - "external_blocked_model_names", - PyFrozenSet::new( - py, - overrides - .getattr("external_blocked_model_names")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ( - "future_cursor_config", - config_or(&config, &["cursors", "future"], none())?, - ), - ( - "start_cursor_config", - config_or(&config, &["cursors", "start"], none())?, - ), - ( - "invocation_time", - runtime_field(runtime, |runtime| &runtime.invocation_time)?, - ), - ("max_microbatches", max_microbatches), - ( - "max_microbatches_is_override", - (!max_override.is_none()) - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ("microbatch_limit_action", limit_action), - ], - )?; + PlanEntryBuildInputs { + source_freshness_blocked_model_names: blocked.unbind(), + external_blocked_model_names: PyFrozenSet::new( + py, + overrides + .getattr("external_blocked_model_names")? + .try_iter()? + .collect::>>()?, + )? + .into_any() + .unbind(), + start_cursor_override: none().unbind(), + end_cursor_override: none().unbind(), + future_cursor_config: config_or(&config, &["cursors", "future"], none())?.unbind(), + start_cursor_config: config_or(&config, &["cursors", "start"], none())?.unbind(), + invocation_time: runtime_field(runtime, |runtime| &runtime.invocation_time)?.unbind(), + max_microbatches: max_microbatches.unbind(), + max_microbatches_is_override: !max_override.is_none(), + microbatch_limit_action: limit_action.unbind(), + }, + )? + .into_any(); plan_entries(PlanEntriesRequest { project, adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 8d08fb586b..a1166b854f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,6 +1,9 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; @@ -34,20 +37,26 @@ pub(crate) fn build_planner_identity_context<'py>( scopes: &Bound<'py, PyAny>, include_stale_warning_identities: bool, ) -> PyResult> { - let identities = version_identities(project, scopes.getattr("inspection_scope")?)?; + let identities = version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + )?; let stale = if include_stale_warning_identities { - version_identities(project, scopes.getattr("stale_warning_scope")?)? + version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + )? } else { identities.clone() }; - native( + Ok(Bound::new( project.py(), - RecordId::PlannerIdentityContext, - &[ - ("version_identities", identities), - ("stale_warning_identities", stale), - ], - ) + PlannerIdentityContext { + version_identities: identities.unbind(), + stale_warning_identities: stale.unbind(), + }, + )? + .into_any()) } fn reusable_model_names( @@ -59,8 +68,12 @@ fn reusable_model_names( if !query_change_tracking { return Ok(names); } - let execution = identities.getattr("version_identities")?; - let stale = identities.getattr("stale_warning_identities")?; + let execution = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; + let stale = class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?; for name in execution_changes.getattr("models")?.try_iter()? { let name = name?; let same = |field: &str| -> PyResult { @@ -122,7 +135,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( .getattr("query_change_tracking")? .is_truthy()?, )?; - let stale_scope = scopes.getattr("stale_warning_scope")?; + let stale_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?; let mut remaining: Vec<_> = Vec::new(); for key in stale_scope .getattr("all_keys")? @@ -136,7 +149,9 @@ pub(crate) fn detect_stale_warning_changes<'py>( remaining.push(key); } } - let stale_identities = identities.getattr("stale_warning_identities")?; + let stale_identities = class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?; let remaining_changes = native( py, NativeAttr::DetectModelChangesInScope, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index b75082aa1c..79bb897f9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod buildability; pub(crate) mod constants; +pub(crate) mod contexts; pub(crate) mod execution_changes; pub(crate) mod full_refresh; pub(crate) mod identities; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index 3df8d67d83..baecdfe807 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -51,3 +51,20 @@ pub(crate) struct PlannerWarehouseState { pub(crate) old_name_views: Py, pub(crate) migration_fingerprints: Py, } + +/// Resolved planner scopes for one plan build. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerScopeResolution { + pub(crate) selected_scope: Py, + pub(crate) stale_warning_scope: Py, + pub(crate) inspection_scope: Py, +} + +/// Expected version identities for inspection and stale-warning scopes. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerIdentityContext { + pub(crate) version_identities: Py, + pub(crate) stale_warning_identities: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index e310d807c3..c6783007b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -4,6 +4,9 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ ChangeReconciliation, ModelEntryResults, PlanOutputFields, }; @@ -75,7 +78,9 @@ pub(crate) fn base_plan_output<'py>( planned_sql_tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let version = identities.getattr("version_identities")?; + let version = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; let extras = native( py, RecordId::PlanOutputExtras, @@ -206,7 +211,7 @@ pub(crate) fn add_storage_policies<'py>( output.field("warnings")?, ], )?; - let selected = scopes.getattr("selected_scope")?; + let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let storage = [ ("runtime", runtime.clone()), ("warehouse", warehouse.clone()), @@ -249,8 +254,14 @@ pub(crate) fn add_plan_warnings<'py>( py, NativeAttr::BuildStaleOutOfSelectionWarnings, &[ - ("original_scope", scopes.getattr("stale_warning_scope")?), - ("execution_scope", scopes.getattr("inspection_scope")?), + ( + "original_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + ), + ( + "execution_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + ), ("changes", stale_warning_changes.clone()), ( "snapshot", @@ -258,7 +269,9 @@ pub(crate) fn add_plan_warnings<'py>( ), ( "version_identities", - identities.getattr("stale_warning_identities")?, + class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?, ), ("source_freshness", source_freshness.clone()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 3ed0682940..5092e097f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -6,6 +6,8 @@ use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!( super::scopes::resolve_planner_scopes, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 2a4666fb15..b7dded0015 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,8 +1,9 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -91,13 +92,13 @@ pub(crate) fn resolve_planner_scopes<'py>( ), ], )?; - native( + Ok(Bound::new( py, - RecordId::PlannerScopeResolution, - &[ - ("selected_scope", selected.clone()), - ("stale_warning_scope", stale_scope), - ("inspection_scope", selected), - ], - ) + PlannerScopeResolution { + selected_scope: selected.clone().unbind(), + stale_warning_scope: stale_scope.unbind(), + inspection_scope: selected.unbind(), + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index 8627463617..47802ad3e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -8,7 +8,9 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -46,7 +48,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( .into_bound(py) .into_any(); progress(runtime, "Inspecting warehouse state...")?; - let selected_scope = scopes.getattr("selected_scope")?; + let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( @@ -76,8 +78,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let stale_keys = pyo3::types::PyFrozenSet::new( py, - scopes - .getattr("stale_warning_scope")? + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)? .getattr("all_keys")? .call_method0("values")? .try_iter()? @@ -109,7 +110,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( project: project.clone(), adapter: adapter.clone(), connection: runtime_field(runtime, |runtime| &runtime.connection)?, - scope: scopes.getattr("inspection_scope")?, + scope: class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, deferral: deferral.clone(), project_config: config, local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index d419863cd5..38d34abd6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -52,21 +52,6 @@ const R_PLANNER_SCOPE: RecordSpec = RecordSpec { ), ], }; -const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerRelationsContext", - doc: "Resolved relation and source inputs for plan entry construction.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_read_map"), - FieldSpec::required("python_source_read_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), - ], -}; const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { name: "SqlTestAssertionStep", doc: "One zero-row assertion step in a SQL-native unit test.", @@ -935,7 +920,6 @@ pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ R_PATH_SELECTOR, R_SQL_TEST_SELECTION, R_PLANNER_SCOPE, - R_PLANNER_RELATIONS_CONTEXT, R_SQL_TEST_ASSERTION_STEP, R_PLANNER_SELECTION, R_PLANNER_OVERRIDES, @@ -1158,27 +1142,6 @@ const R_UNIT_TEST_SELECTOR_SPLIT: RecordSpec = RecordSpec { FieldSpec::required("sql_test_selection"), ], }; -const R_MODEL_PLAN_CONTEXT: RecordSpec = RecordSpec { - name: "ModelPlanContext", - doc: "Relation lookups and source columns for building one model plan entry.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("models_by_name"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("functions_by_name", FieldDefault::EmptyDict), - FieldSpec::new( - "runtime_cursor_producer_names", - FieldDefault::EmptyFrozenset, - ), - FieldSpec::new("future_cursor_config", FieldDefault::None), - FieldSpec::new("start_cursor_config", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::None), - ], -}; const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { name: "CursorOverridePair", doc: "Resolved per-model cursor start/end overrides for plan entry construction.", @@ -1187,28 +1150,6 @@ const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { FieldSpec::new("end_cursor_override", FieldDefault::None), ], }; -const R_PLAN_ENTRY_BUILD_INPUTS: RecordSpec = RecordSpec { - name: "PlanEntryBuildInputs", - doc: "Blocked models and cursor overrides for plan entry building.", - fields: &[ - FieldSpec::new( - "source_freshness_blocked_model_names", - FieldDefault::EmptyFrozenset, - ), - FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyFrozenset), - FieldSpec::new("start_cursor_override", FieldDefault::None), - FieldSpec::new("end_cursor_override", FieldDefault::None), - FieldSpec::new("future_cursor_config", FieldDefault::None), - FieldSpec::new("start_cursor_config", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::None), - FieldSpec::new("max_microbatches", FieldDefault::None), - FieldSpec::new("max_microbatches_is_override", FieldDefault::False), - FieldSpec::new( - "microbatch_limit_action", - FieldDefault::NativeMember(RecordId::MicrobatchLimitAction, "ERROR"), - ), - ], -}; const R_FUNCTION_CHANGE_RESULT: RecordSpec = RecordSpec { name: "FunctionChangeResult", doc: "Per-function output from change detection.", @@ -1394,15 +1335,6 @@ const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { FieldSpec::new("annotation_module", FieldDefault::None), ], }; -const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { - name: "PlannerScopeResolution", - doc: "Resolved planner scopes for one plan build.", - fields: &[ - FieldSpec::required("selected_scope"), - FieldSpec::required("stale_warning_scope"), - FieldSpec::required("inspection_scope"), - ], -}; const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { name: "ModelMigrationRequest", doc: "One destination model whose data should come from an earlier relation.", @@ -1442,14 +1374,6 @@ const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { FieldSpec::new("findings", FieldDefault::EmptyTuple), ], }; -const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerIdentityContext", - doc: "Expected version identities for inspection and stale-warning scopes.", - fields: &[ - FieldSpec::required("version_identities"), - FieldSpec::required("stale_warning_identities"), - ], -}; /// Planner value records, in registration order. pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1469,9 +1393,7 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_BACKFILL_RESULT, R_CHANGE_DETECTION_RESULT, R_UNIT_TEST_SELECTOR_SPLIT, - R_MODEL_PLAN_CONTEXT, R_CURSOR_OVERRIDE_PAIR, - R_PLAN_ENTRY_BUILD_INPUTS, R_FUNCTION_CHANGE_RESULT, R_PLANNER_CHANGE_RESULTS, R_RESOLVED_MODEL_ACTION, @@ -1490,12 +1412,10 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_SCENARIO_ARTIFACT_IDENTITY, R_SCENARIO_ARTIFACT_NAME, R_PLAN_PROVIDER_USAGE, - R_PLANNER_SCOPE_RESOLUTION, R_MODEL_MIGRATION_REQUEST, R_MODEL_MIGRATION_DECLARATION, R_MODEL_MIGRATION_DISCOVERY, R_MIGRATION_COMPATIBILITY_RESULT, - R_PLANNER_IDENTITY_CONTEXT, ]; /// Planner string enums, in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 6d8260bb67..f28f68af41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -8,6 +8,8 @@ use crate::bindings::_helpers::model_config::config_values::{ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, @@ -76,8 +78,14 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(py.None().into_bound(py)); } let incremental_mode = config(&model, "incremental_mode")?; - let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; - let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; + let future_config = future_cursor_config( + &model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?; + let start_config = start_cursor_config( + &model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?; if incremental_mode.eq("microbatch")? { return unbounded(py); } @@ -104,7 +112,9 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, &[("model", model.clone())], )?; - let producers = context.getattr("runtime_cursor_producer_names")?; + let producers = class_field(context, |c: &ModelPlanContext| { + &c.runtime_cursor_producer_names + })?; let mut runtime_owned = false; for name in roles.getattr("watermark_inputs")?.try_iter()? { if producers.contains(name?)? { @@ -196,7 +206,10 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ®istry::class(py, RecordId::MaximumStartPolicyInputs)?, &[ ("config", start_config), - ("invocation_time", context.getattr("invocation_time")?), + ( + "invocation_time", + class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, + ), ( "incremental_strategy", config(&model, "incremental_strategy")?, @@ -232,7 +245,10 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ("cursor_type", cursor_type), ("cursor_grain", cursor_grain), ("config", future_config), - ("invocation_time", context.getattr("invocation_time")?), + ( + "invocation_time", + class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, + ), ( "has_complete_override", (!start_override.is_none() && !end_override.is_none()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index da8a171299..2916eae334 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -10,6 +10,8 @@ use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, @@ -526,8 +528,14 @@ pub(crate) fn cursor_safety_configs<'py>( context: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { Ok(( - future_cursor_config(model, &context.getattr("future_cursor_config")?)?, - start_cursor_config(model, &context.getattr("start_cursor_config")?)?, + future_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?, + start_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 81d9bbcea8..d27886200c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,6 +9,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; use crate::bindings::_helpers::planner_resolution::refs::{ @@ -43,11 +45,13 @@ impl<'py> Resolution<'py> { Ok(Self { adapter, external, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - function_locations: context.getattr("function_locations")?, + source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, + source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { + &c.source_warehouse_columns + })?, + model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, + seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, + function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 1303f5e260..c96c3b41d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -5,6 +5,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; use crate::bindings::_helpers::planner_scenario::{ entries::{function_entry, seed_entries}, @@ -18,7 +19,7 @@ use crate::bindings::_helpers::planner_strategy::fixture_context::planning_conte use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -250,31 +251,30 @@ fn model_context<'py>( ) -> PyResult> { let (models, locations, columns) = inputs; let py = project.py(); - call_with( - ®istry::class(py, RecordId::ModelPlanContext)?, - &[ - ( - "model_locations", - context.relations.getattr("model_locations")?, - ), - ("models_by_name", models.clone().into_any()), - ( - "functions_by_name", - by_attribute(&project.getattr("functions")?, "name")?.into_any(), - ), - ( - "seed_locations", - context.relations.getattr("seed_locations")?, - ), - ("function_locations", locations.clone()), - ("source_map", context.relations.getattr("source_map")?), - ("source_warehouse_columns", columns.clone()), - ( - "star_exclude_keyword", - context.adapter.call_method0("star_exclude_keyword")?, - ), - ], - ) + let none = || py.None(); + Ok(Bound::new( + py, + ModelPlanContext { + model_locations: context.relations.getattr("model_locations")?.unbind(), + models_by_name: models.clone().into_any().unbind(), + functions_by_name: by_attribute(&project.getattr("functions")?, "name")? + .into_any() + .unbind(), + seed_locations: context.relations.getattr("seed_locations")?.unbind(), + function_locations: locations.clone().unbind(), + source_map: context.relations.getattr("source_map")?.unbind(), + source_warehouse_columns: columns.clone().unbind(), + star_exclude_keyword: context + .adapter + .call_method0("star_exclude_keyword")? + .unbind(), + runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), + future_cursor_config: none(), + start_cursor_config: none(), + invocation_time: none(), + }, + )? + .into_any()) } fn expectations<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 4926d5bdc0..61cf4bd37d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -337,7 +337,6 @@ native_ids! { ModelMigrationDiscovery => "ModelMigrationDiscovery", ModelMigrationPlanEntry => "ModelMigrationPlanEntry", ModelMigrationRequest => "ModelMigrationRequest", - ModelPlanContext => "ModelPlanContext", ModelPlanEntry => "ModelPlanEntry", ModelSchemaDeclaration => "ModelSchemaDeclaration", ModelSql => "ModelSql", @@ -381,7 +380,6 @@ native_ids! { PathSelector => "PathSelector", PlanAction => "PlanAction", PlanCommandRequest => "PlanCommandRequest", - PlanEntryBuildInputs => "PlanEntryBuildInputs", PlanOutput => "PlanOutput", PlanOutputExtras => "PlanOutputExtras", PlanProviderUsage => "PlanProviderUsage", @@ -390,13 +388,10 @@ native_ids! { PlanWarning => "PlanWarning", PlannedSqlTests => "PlannedSqlTests", PlannerChangeResults => "PlannerChangeResults", - PlannerIdentityContext => "PlannerIdentityContext", PlannerOverrides => "PlannerOverrides", PlannerPolicies => "PlannerPolicies", - PlannerRelationsContext => "PlannerRelationsContext", PlannerResolvedActions => "PlannerResolvedActions", PlannerScope => "PlannerScope", - PlannerScopeResolution => "PlannerScopeResolution", PlannerSelection => "PlannerSelection", PlaygroundCommandRequest => "PlaygroundCommandRequest", PlaygroundTemplate => "PlaygroundTemplate", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4504f51c5a..34b4a4c8dc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -980,7 +980,7 @@ class UnitTestSelectorSplit(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> UnitTestSelectorSplit: ... -class ModelPlanContext(NativeRecord): +class ModelPlanContext: """Relation lookups and source columns for building one model plan entry.""" model_locations: dict[str, CompiledRelationLocation] @@ -997,6 +997,7 @@ class ModelPlanContext(NativeRecord): invocation_time: datetime | None def __init__( self, + *, model_locations: dict[str, CompiledRelationLocation], models_by_name: dict[str, CompiledModelView], seed_locations: dict[str, CompiledRelationLocation], @@ -1010,7 +1011,6 @@ class ModelPlanContext(NativeRecord): start_cursor_config: StartCursorsConfig | None = ..., invocation_time: datetime | None = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> ModelPlanContext: ... class CursorOverridePair(NativeRecord): """Resolved per-model cursor start/end overrides for plan entry construction.""" @@ -1022,7 +1022,7 @@ class CursorOverridePair(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> CursorOverridePair: ... -class PlanEntryBuildInputs(NativeRecord): +class PlanEntryBuildInputs: """Blocked models and cursor overrides for plan entry building.""" source_freshness_blocked_model_names: frozenset[str] @@ -1037,6 +1037,7 @@ class PlanEntryBuildInputs(NativeRecord): microbatch_limit_action: MicrobatchLimitAction def __init__( self, + *, source_freshness_blocked_model_names: frozenset[str] = ..., external_blocked_model_names: frozenset[str] = ..., start_cursor_override: str | None = ..., @@ -1048,7 +1049,6 @@ class PlanEntryBuildInputs(NativeRecord): max_microbatches_is_override: bool = ..., microbatch_limit_action: MicrobatchLimitAction = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlanEntryBuildInputs: ... class FunctionChangeResult(NativeRecord): """Per-function output from change detection.""" @@ -1340,7 +1340,7 @@ class PlannerRuntime: invocation_time: datetime = ..., ) -> None: ... -class PlannerScopeResolution(NativeRecord): +class PlannerScopeResolution: """Resolved planner scopes for one plan build.""" selected_scope: PlannerScope @@ -1348,11 +1348,11 @@ class PlannerScopeResolution(NativeRecord): inspection_scope: PlannerScope def __init__( self, + *, selected_scope: PlannerScope, stale_warning_scope: PlannerScope, inspection_scope: PlannerScope, ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... class PlannerWarehouseState: """Warehouse snapshot and inspection relations gathered once per plan.""" @@ -1435,17 +1435,17 @@ class MigrationCompatibilityResult(NativeRecord): def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... -class PlannerIdentityContext(NativeRecord): +class PlannerIdentityContext: """Expected version identities for inspection and stale-warning scopes.""" version_identities: DirectModelVersionIdentities stale_warning_identities: DirectModelVersionIdentities def __init__( self, + *, version_identities: DirectModelVersionIdentities, stale_warning_identities: DirectModelVersionIdentities, ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... class NativeModelInput: @property @@ -12300,7 +12300,7 @@ class PlannerScope(NativeRecord): sql_test_selection: SqlTestSelection = ..., ) -> None: ... -class PlannerRelationsContext(NativeRecord): +class PlannerRelationsContext: """Resolved relation and source inputs for plan entry construction.""" model_locations: dict[str, CompiledRelationLocation] @@ -12314,6 +12314,7 @@ class PlannerRelationsContext(NativeRecord): listed_source_names: frozenset[str] def __init__( self, + *, model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], function_locations: dict[str, CompiledRelationLocation], diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 88f507dd64..101fbd5829 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -10,6 +10,7 @@ CompiledRelationLocation, CompileModelConfig, ModelPlanContext, + SourceEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -34,16 +35,21 @@ def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> Compile ) -def build_empty_model_plan_context() -> ModelPlanContext: - """Build an empty model-resolution context.""" +def build_empty_model_plan_context( + *, + model_locations: dict[str, CompiledRelationLocation] | None = None, + seed_locations: dict[str, CompiledRelationLocation] | None = None, + source_map: dict[str, SourceEntry] | None = None, +) -> ModelPlanContext: + """Build a model-resolution context with only the given relations.""" return ModelPlanContext( - model_locations={}, + model_locations=model_locations or {}, models_by_name={}, functions_by_name={}, - seed_locations={}, + seed_locations=seed_locations or {}, function_locations={}, - source_map={}, + source_map=source_map or {}, source_warehouse_columns={}, star_exclude_keyword="EXCLUDE", ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index c00449ebb0..461182ea37 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -2,8 +2,6 @@ from __future__ import annotations -from dataclasses import replace - import pytest from sqlbuild._native import ( @@ -25,8 +23,7 @@ ) _PYTHON_HOOK: PythonHookEntry = PythonHookEntry(name="refresh_lookup", kwargs={}) -_CONTEXT: ModelPlanContext = replace( - build_empty_model_plan_context(), +_CONTEXT: ModelPlanContext = build_empty_model_plan_context( model_locations={"orders": build_target("staging.orders", "orders")}, seed_locations={"country_codes": build_target("seeds.country_codes", "country_codes")}, source_map={"raw_orders": SourceEntry(name="raw_orders", schema="raw_prod", table="orders")}, From a1e5058294e0e48476a09c080bbcd7578d0417ff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:03:52 +0100 Subject: [PATCH 2/4] refactor(migrations): plan model migrations over Rust requests Model migration requests, declarations, discovery results and compatibility results are Rust structs. Discovery, fingerprint matching, origin resolution, compatibility checks, decisions, execution choices, redirected cursor inspection and the live-state inspection are called directly instead of through the native module. Removes the corresponding native records and pyfunctions, none of which had a Python caller. --- .../planner_migrations/column_overlay.rs | 21 +- .../planner_migrations/column_planning.rs | 23 +- .../planner_migrations/compatibility.rs | 59 ++--- .../_helpers/planner_migrations/discovery.rs | 174 +++++------- .../planner_migrations/discovery_events.rs | 14 +- .../planner_migrations/discovery_requests.rs | 43 ++- .../_helpers/planner_migrations/matching.rs | 149 ++++------- .../planner_migrations/model_entries.rs | 110 ++++---- .../planner_migrations/model_handover.rs | 5 +- .../planner_migrations/model_overlay.rs | 18 +- .../planner_migrations/model_planning.rs | 249 ++++++------------ .../_helpers/planner_migrations/models.rs | 79 ++++++ .../planner_migrations/old_name_planning.rs | 29 +- .../planner_migrations/planning_boundary.rs | 90 +++---- .../planner_migrations/planning_execution.rs | 15 +- .../planner_migrations/planning_origin.rs | 64 ++--- .../planner_migrations/registration.rs | 6 - .../planner_migrations/state_inspection.rs | 15 ++ .../_helpers/planner_records/constants.rs | 43 --- .../src/bindings/_helpers/records/ids.rs | 18 -- src/sqlbuild/_native.pyi | 131 --------- 21 files changed, 454 insertions(+), 901 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 1126347f21..f1743b77de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,7 +1,7 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; @@ -10,7 +10,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -161,20 +160,10 @@ pub(super) fn cursors<'py>( }; targets.set_item(name, target)?; } - let redirected = call_with( - ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, - &[ - ("runtime", phase.runtime.clone()), - ("scope", scope), - ( - "existing_relations", - snapshot.getattr("existing_relations")?, - ), - ("target_relations", targets.into_any()), - ("overrides", phase.overrides.clone()), - ("deferral", phase.deferral.clone()), - ("origin_cursor_columns", origins.into_any()), - ], + let redirected = super::planning_boundary::redirected_cursor_snapshots( + phase, + (snapshot.getattr("existing_relations")?, targets.into_any()), + Some(origins.into_any()), )?; let cursors = imported(py, "builtins", "dict")?.call1((snapshot.getattr("cursor_snapshots")?,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 929aa2a268..66423a9c63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -8,8 +8,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -77,21 +75,7 @@ pub(crate) fn plan_column_migrations<'py>( warnings: Vec::new(), }); } - let database = call_with( - ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, - &[("runtime", runtime.clone())], - )?; - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ("adapter", adapter), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ("database", database), - ], - )?; + let state = super::state_inspection::inspection(&runtime, None)?; let schemas = PySet::empty(py)?; for candidate in &candidates { let schema = model_reads::destination(&candidate.model)?.getattr("schema")?; @@ -131,10 +115,7 @@ pub(crate) fn plan_column_migrations<'py>( entries.extend(checked.entries); warnings.extend(checked.warnings); } - let policy = call_with( - ®istry::native(py, NativeAttr::MigrationMissingOriginPolicy)?, - &[("runtime", runtime)], - )?; + let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for entry in &mut entries { *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; if let Some(warning) = super::column_warnings::entry_warning(entry)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 00f3fa21a4..d081e0df53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -3,6 +3,9 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::planner_migrations::models::{ + CompatibilityCheck, MigrationCompatibilityResult, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::exceptions::PyValueError; @@ -106,30 +109,13 @@ fn migration_snapshot_blocking_findings<'py>( ) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn check_migration_compatibility<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "check_migration_compatibility", - options, - ( - &[ - "model", - "origin_relation", - "origin_columns", - "sql_analysis_enabled", - "column_dialect", - "snapshots_config", - ], - &[], - ), - )?; - let model = arguments.value("model")?; - let relation = arguments.value("origin_relation")?; +/// Whether an origin relation's columns can carry the destination model's history. +pub(super) fn check_migration_compatibility<'py>( + check: &CompatibilityCheck<'py>, +) -> PyResult> { + let py = check.model.py(); + let model = check.model.clone(); + let relation = check.origin_relation.clone(); let relation_type: String = relation.getattr("relation_type")?.extract()?; if relation_type.to_uppercase().contains("VIEW") { return result( @@ -146,14 +132,14 @@ fn check_migration_compatibility<'py>( let columns = PyDict::new(py); columns.set_item( model_reads::name_object(&model)?, - arguments.value("origin_columns")?, + check.origin_columns.clone(), )?; let snapshot = call_with( ®istry::class(py, RecordId::WarehouseSnapshot)?, &[ ("existing_relations", relations.into_any()), ("existing_columns", columns.into_any()), - ("column_dialect", arguments.value("column_dialect")?), + ("column_dialect", check.column_dialect.clone()), ], )?; let change = call_with( @@ -161,10 +147,7 @@ fn check_migration_compatibility<'py>( &[ ("model", model.clone()), ("snapshot", snapshot), - ( - "sql_analysis_enabled", - arguments.value("sql_analysis_enabled")?, - ), + ("sql_analysis_enabled", check.sql_analysis_enabled.clone()), ( "query_change_tracking", false.into_pyobject(py)?.to_owned().into_any(), @@ -186,10 +169,10 @@ fn check_migration_compatibility<'py>( return result(py, "not_checked", PyTuple::empty(py)); } let snapshot = config(&model, "materialized")?.as_deref() == Some("snapshot"); - let snapshots_config = arguments.value("snapshots_config")?; + let snapshots_config = check.snapshots_config.clone(); let findings = blocking( (&model, &change.getattr("schema_findings")?), - &arguments.value("column_dialect")?, + &check.column_dialect, snapshot.then_some(&snapshots_config), )?; result( @@ -207,16 +190,14 @@ fn result<'py>( py: Python<'py>, status: &str, findings: Bound<'py, PyTuple>, -) -> PyResult> { - let status = registry::class(py, RecordId::MigrationCompatibility)?.call1((status,))?; - call_with( - ®istry::class(py, RecordId::MigrationCompatibilityResult)?, - &[("status", status), ("findings", findings.into_any())], - ) +) -> PyResult> { + Ok(MigrationCompatibilityResult { + status: registry::class(py, RecordId::MigrationCompatibility)?.call1((status,))?, + findings: findings.into_any(), + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_migration_compatibility, module)?)?; module.add_function(wrap_pyfunction!( migration_incremental_blocking_findings, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index 24535fae12..923c24df27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,61 +1,58 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_migrations::matching::{ + match_migration_fingerprints, migration_destination_fingerprints, +}; +use crate::bindings::_helpers::planner_migrations::models::{ + DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, +}; use crate::bindings::_helpers::planner_migrations::{ discovery_events, discovery_models, discovery_requests, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; +use std::collections::BTreeSet; -#[pyfunction] -#[pyo3(signature = (**options))] -fn discover_model_migrations<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "discover_model_migrations", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "manual_requests", - "declarations", - "state", - "project_schemas", - "fingerprints", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; - let state = args.value("state")?; - let manual = args.value("manual_requests")?; - let declarations = args.value("declarations")?; +/// Manual and automatically discovered migration requests with their destination fingerprints. +pub(super) fn discover_model_migrations<'py>( + request: DiscoveryRequest<'py>, +) -> PyResult> { + let DiscoveryRequest { + runtime, + scope, + snapshot, + manual_requests: manual, + declarations, + state, + project_schemas, + fingerprints: cache, + } = request; + let py = runtime.py(); let models = discovery_models::ordered(&runtime)?; let declared_names = PyDict::new(py); - for declaration in declarations.try_iter()? { - declared_names.set_item(declaration?.getattr("model_name")?, py.None())?; + for declaration in &declarations { + declared_names.set_item(&declaration.model_name, py.None())?; } - let triggered = - discovery_models::triggered(&models, &scope, &args.value("snapshot")?, &declared_names)?; - if !triggered && !manual.is_truthy()? { - return native(py, RecordId::ModelMigrationDiscovery, &[]); + let triggered = discovery_models::triggered(&models, &scope, &snapshot, &declared_names)?; + if !triggered && manual.is_empty() { + return Ok(ModelMigrationDiscovery { + requests: Vec::new(), + warnings: Vec::new(), + destination_fingerprints: PyDict::new(py), + }); } let metadata = discovery_models::metadata(&runtime, &models)?; - let dialect = runtime - .getattr("adapter")? - .call_method0("sql_analysis_dialect")?; + let inputs = FingerprintInputs { + metadata_jsons: metadata.into_any(), + dialect: runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?, + fingerprints: cache, + }; call_with( &state.getattr("inspect_schemas")?, - &[("schemas", args.value("project_schemas")?)], + &[("schemas", project_schemas)], )?; let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); @@ -66,25 +63,15 @@ fn discover_model_migrations<'py>( for (name, origin) in declared.iter() { event_renames.set_item(name, origin)?; } - let seeds = PyTuple::new(py, [event_renames, declared])?; - let cache = args.value("fingerprints")?; + let seeds = PyTuple::new(py, [event_renames, declared])?.into_any(); if !triggered { - let hashes = native( - py, - NativeAttr::MigrationDestinationFingerprints, - &[ - ("requests", manual.clone()), - ("variants", seeds.into_any()), - ("metadata_jsons", metadata.into_any()), - ("dialect", dialect), - ("fingerprints", cache), - ], - )?; - return native( - py, - RecordId::ModelMigrationDiscovery, - &[("requests", manual), ("destination_fingerprints", hashes)], - ); + let destination_fingerprints = + migration_destination_fingerprints(&manual, &seeds, &inputs)?; + return Ok(ModelMigrationDiscovery { + requests: manual, + warnings: Vec::new(), + destination_fingerprints, + }); } let unbuilt = PyDict::new(py); for item in &models { @@ -105,53 +92,26 @@ fn discover_model_migrations<'py>( for (name, _) in resumed.iter() { unbuilt.del_item(name)?; } - let matched = native( - py, - NativeAttr::MatchMigrationFingerprints, - &[ - ("models", PyTuple::new(py, &models)?.into_any()), - ( - "new_names", - PyFrozenSet::new(py, unbuilt.keys().iter())?.into_any(), - ), - ("seed_variants", seeds.into_any()), - ("candidates", candidates.into_any()), - ("metadata_jsons", metadata.clone().into_any()), - ("dialect", dialect.clone()), - ("fingerprints", cache.clone()), - ], - )?; - let mut requests: Vec> = manual.try_iter()?.collect::>()?; + let new_names: BTreeSet = unbuilt + .keys() + .iter() + .map(|name| name.extract()) + .collect::>()?; + let (variant_maps, matches, ambiguous) = + match_migration_fingerprints((&models, new_names), &seeds, candidates.as_any(), &inputs)?; + let mut requests = manual; requests.extend(discovery_requests::automatic( &runtime, &scope, &models, - (&matched.get_item(1)?, &resumed), + (matches.as_any(), &resumed), )?); - let requests = PyTuple::new(py, requests)?; - let warnings = discovery_requests::warnings(&models, &scope, &matched.get_item(2)?)?; - let hashes = native( - py, - NativeAttr::MigrationDestinationFingerprints, - &[ - ("requests", requests.clone().into_any()), - ("variants", matched.get_item(0)?), - ("metadata_jsons", metadata.into_any()), - ("dialect", dialect), - ("fingerprints", cache), - ], - )?; - native( - py, - RecordId::ModelMigrationDiscovery, - &[ - ("requests", requests.into_any()), - ("warnings", warnings.into_any()), - ("destination_fingerprints", hashes), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_migrations, module)?) + let warnings = discovery_requests::warnings(&models, &scope, &ambiguous)?; + let destination_fingerprints = + migration_destination_fingerprints(&requests, &variant_maps, &inputs)?; + Ok(ModelMigrationDiscovery { + requests, + warnings: warnings.iter().collect(), + destination_fingerprints, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 07254edf57..5b1e8635ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::config; +use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; @@ -32,20 +33,19 @@ pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult( - declarations: &Bound<'py, PyAny>, + declarations: &[ModelMigrationDeclaration<'py>], state: &Bound<'py, PyAny>, ) -> PyResult> { let result = PyDict::new(state.py()); - for item in declarations.try_iter()? { - let item = item?; + for item in declarations { let fingerprint = call_with( &state.getattr("model_fingerprint")?, &[ - ("location", item.getattr("origin_location")?), - ("model_name", item.getattr("origin_model")?), + ("location", item.origin_location.clone()), + ("model_name", item.origin_model.clone()), ], )?; - let explicit = item.getattr("origin_model")?; + let explicit = item.origin_model.clone(); let origin = if explicit.is_truthy()? { explicit } else if !fingerprint.is_none() { @@ -54,7 +54,7 @@ pub(super) fn declared<'py>( state.py().None().into_bound(state.py()) }; if !origin.is_none() { - result.set_item(item.getattr("model_name")?, origin)?; + result.set_item(&item.model_name, origin)?; } } Ok(result) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index 205e96846b..9f6e729988 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -6,12 +6,14 @@ use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ identity, key, newest, relation, }; +use crate::bindings::_helpers::planner_migrations::models::{ + ModelMigrationDeclaration, ModelMigrationRequest, +}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, + PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, }; use sqlbuild_analysis::planner_migrations::main::discovery_materialization::discovery_materialization; use sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint; @@ -48,12 +50,12 @@ pub(super) fn candidates<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, - claims: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), + claims: (&[ModelMigrationDeclaration<'py>], &Bound<'py, PyDict>), ) -> PyResult> { let py = runtime.py(); let excluded = PySet::empty(py)?; - for declaration in claims.0.try_iter()? { - excluded.add(key(&declaration?.getattr("origin_location")?)?)?; + for declaration in claims.0 { + excluded.add(key(&declaration.origin_location)?)?; } for (_, event) in claims.1.iter() { excluded.add(key(&event.getattr("origin")?)?)?; @@ -107,9 +109,9 @@ pub(super) fn automatic<'py>( scope: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], matches: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), -) -> PyResult>> { +) -> PyResult>> { let py = runtime.py(); - let mut requests: Vec> = Vec::new(); + let mut requests: Vec> = Vec::new(); for item in models { if !scope .getattr("selected_keys")? @@ -140,23 +142,16 @@ pub(super) fn automatic<'py>( None }; if let Some((location, origin_model)) = origin { - requests.push(native( - py, - RecordId::ModelMigrationRequest, - &[ - ("model", item.clone()), - ( - "discovery", - registry::class(py, RecordId::MigrationDiscovery)?.getattr("AUTOMATIC")?, - ), - ("origin_location", location), - ("origin_model", origin_model), - ( - "identity_only", - PyBool::new(py, identity(item)?).to_owned().into_any(), - ), - ], - )?); + requests.push(ModelMigrationRequest { + model: item.clone(), + discovery: registry::class(py, RecordId::MigrationDiscovery)? + .getattr("AUTOMATIC")?, + raw_origin: None, + origin_location: Some(location), + origin_model, + force: false, + identity_only: identity(item)?, + }); } } Ok(requests) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index f8106545b7..592daf36bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,23 +1,25 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_migrations::models::{ + FingerprintInputs, ModelMigrationRequest, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::match_migrations::match_migrations; use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; use std::collections::{BTreeMap, BTreeSet}; fn variants<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, + inputs: &FingerprintInputs<'py>, model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { let name = model_reads::name_object(model)?; - let metadata = arguments.value("metadata_jsons")?.get_item(name)?; - let cache = arguments.value("fingerprints")?.getattr("fingerprint")?; + let metadata = inputs.metadata_jsons.get_item(name)?; + let cache = inputs.fingerprints.getattr("fingerprint")?; let mut hashes: Vec = Vec::new(); for map in maps.try_iter()? { let hash: Option = call_with( @@ -26,7 +28,7 @@ fn variants<'py>( ("query_sql", model_reads::query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), - ("dialect", arguments.value("dialect")?), + ("dialect", inputs.dialect.clone()), ], )? .extract()?; @@ -36,105 +38,62 @@ fn variants<'py>( hashes.push(hash); } } - let _ = py; Ok(hashes) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn match_migration_fingerprints<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "match_migration_fingerprints", - options, - ( - &[ - "models", - "new_names", - "seed_variants", - "candidates", - "metadata_jsons", - "dialect", - "fingerprints", - ], - &[], - ), - )?; - let new_names: BTreeSet = arguments.value("new_names")?.extract()?; - let mut models: BTreeMap> = BTreeMap::new(); +/// The fingerprint variants tried, the matched candidate per new model, and ambiguous names. +pub(super) fn match_migration_fingerprints<'py>( + (models, new_names): (&[Bound<'py, PyAny>], BTreeSet), + seed_variants: &Bound<'py, PyAny>, + candidates: &Bound<'py, PyAny>, + inputs: &FingerprintInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyAny>)> { + let py = seed_variants.py(); + let mut by_name: BTreeMap> = BTreeMap::new(); let mut names: Vec = Vec::new(); - for model in arguments.value("models")?.try_iter()? { - let model = model?; - let name: String = model_reads::model_name(&model)?; + for model in models { + let name: String = model_reads::model_name(model)?; if new_names.contains(&name) { names.push(name.clone()); } - models.insert(name, model); + by_name.insert(name, model.clone()); } - let original: Vec> = arguments - .value("candidates")? - .try_iter()? - .collect::>()?; - let mut candidates: Vec = Vec::new(); + let original: Vec> = candidates.try_iter()?.collect::>()?; + let mut fingerprints: Vec = Vec::new(); for candidate in &original { - candidates.push(MigrationCandidate { + fingerprints.push(MigrationCandidate { name: candidate.getattr("node_name")?.extract()?, fingerprint: stored_migration_fingerprint(candidate)?.unwrap_or_default(), }); } - let seeds: Vec> = arguments.value("seed_variants")?.extract()?; - let matched = match_migrations(&names, &seeds, &candidates, |name, maps| { - variants( - py, - &arguments, - &models[name], - PyTuple::new(py, maps)?.into_any(), - ) + let seeds: Vec> = seed_variants.extract()?; + let matched = match_migrations(&names, &seeds, &fingerprints, |name, maps| { + variants(inputs, &by_name[name], PyTuple::new(py, maps)?.into_any()) })?; let matches = PyDict::new(py); for (name, index) in matched.matches { matches.set_item(name, &original[index])?; } - PyTuple::new( - py, - [ - PyTuple::new(py, matched.variants)?.into_any(), - matches.into_any(), - matched.ambiguous.into_pyobject(py)?.into_any(), - ], - ) + Ok(( + PyTuple::new(py, matched.variants)?.into_any(), + matches, + matched.ambiguous.into_pyobject(py)?.into_any(), + )) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn migration_destination_fingerprints<'py>( - py: Python<'py>, - options: Option>, +/// The fingerprint variants of each requested destination model with metadata. +pub(super) fn migration_destination_fingerprints<'py>( + requests: &[ModelMigrationRequest<'py>], + variant_maps: &Bound<'py, PyAny>, + inputs: &FingerprintInputs<'py>, ) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "migration_destination_fingerprints", - options, - ( - &[ - "requests", - "variants", - "metadata_jsons", - "dialect", - "fingerprints", - ], - &[], - ), - )?; + let py = variant_maps.py(); let result = PyDict::new(py); - for request in arguments.value("requests")?.try_iter()? { - let model = request?.getattr("model")?; - let name = model_reads::name_object(&model)?; - if arguments - .value("metadata_jsons")? + for request in requests { + let model = &request.model; + let name = model_reads::name_object(model)?; + if inputs + .metadata_jsons .call_method1("get", (&name,))? .is_none() { @@ -142,26 +101,16 @@ fn migration_destination_fingerprints<'py>( } result.set_item( name, - PyTuple::new( - py, - variants(py, &arguments, &model, arguments.value("variants")?)?, - )?, + PyTuple::new(py, variants(inputs, model, variant_maps.clone())?)?, )?; } Ok(result) } -#[pyfunction] -fn stored_migration_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult> { +/// The migration fingerprint stored in a model fingerprint's metadata. +pub(super) fn stored_migration_fingerprint( + fingerprint: &Bound<'_, PyAny>, +) -> PyResult> { let metadata: String = fingerprint.getattr("metadata_json")?.extract()?; Ok(sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint(&metadata)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(match_migration_fingerprints, module)?)?; - module.add_function(wrap_pyfunction!( - migration_destination_fingerprints, - module - )?)?; - module.add_function(wrap_pyfunction!(stored_migration_fingerprint, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 59a4408c60..aa28283afe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -1,19 +1,21 @@ //! Assemble model migration decisions from inspected warehouse facts. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_migrations::column_objects::{ - lower, model, native, replace, text, -}; +use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; +use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; use crate::bindings::_helpers::planner_migrations::model_handover::{ applies, empty_version, handover, }; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_migrations::models::{ + CompatibilityCheck, MigrationDecisionRequest, +}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyBool, PyTuple}; +use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { @@ -44,30 +46,35 @@ pub(super) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult( entry: Bound<'py, PyAny>, - fingerprint: Bound<'py, PyAny>, + inherited: Bound<'py, PyAny>, applies: bool, -) -> PyResult> { - let py = entry.py(); - Ok(PyTuple::new( - py, - [ - entry, - fingerprint, - PyBool::new(py, applies).to_owned().into_any(), - ], - )? - .into_any()) +) -> PyResult> { + Ok(MigrationDecision { + entry, + inherited, + applies, + }) +} + +/// The decided plan entry, the inherited fingerprint, and whether the history handover applies. +pub(super) struct MigrationDecision<'py> { + pub(super) entry: Bound<'py, PyAny>, + pub(super) inherited: Bound<'py, PyAny>, + pub(super) applies: bool, } -pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult> { - let runtime = args.value("runtime")?; +pub(super) fn decide<'py>( + decision_request: &MigrationDecisionRequest<'_, 'py>, +) -> PyResult> { + let runtime = decision_request.runtime.clone(); let py = runtime.py(); - let request = args.value("request")?; - let item = request.getattr("model")?; + let request = decision_request.request; + let item = request.model.clone(); let destination = item.getattr("destination")?; - let origin = args.value("origin")?; - let snapshot = args.value("snapshot")?; - let state = args.value("state")?; + let origin = decision_request.origin.clone(); + let snapshot = decision_request.snapshot.clone(); + let state = decision_request.state.clone(); + let origin_model = decision_request.origin_model.clone(); if key(&origin)? == key(&destination)? { let name: String = item.getattr("name")?.extract()?; return Err(pyo3::PyErr::from_value( @@ -82,7 +89,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult( ) -> PyResult> { let py = model.py(); if handover.is_none() - || !destinations.contains( - registry::native(py, NativeAttr::StoredMigrationFingerprint)?.call1((&handover,))?, - )? + || !destinations.contains(super::matching::stored_migration_fingerprint(&handover)?)? { return Ok(handover); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index 6d8cca941a..1b9941282b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -1,12 +1,10 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -89,16 +87,10 @@ pub(super) fn overlay<'py>( .getattr("cursor_snapshots")? .cast_into::()? .copy()?; - let gathered = call_with( - ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, - &[ - ("runtime", runtime), - ("scope", phase.scope.clone()), - ("existing_relations", relations.clone().into_any()), - ("target_relations", redirected.into_any()), - ("overrides", phase.overrides.clone()), - ("deferral", phase.deferral.clone()), - ], + let gathered = super::planning_boundary::redirected_cursor_snapshots( + phase, + (relations.clone().into_any(), redirected.into_any()), + None, )?; cursors.call_method1("update", (gathered,))?; replace( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 515c7577c7..8a9849da3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -1,20 +1,24 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; +use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; use crate::bindings::_helpers::planner_migrations::models::{ - MigrationPhase, ModelMigrationPlanning, + DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, + ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, +}; +use crate::bindings::_helpers::planner_migrations::planning_origin::{ + require_same_database, resolve_origin, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple, }; -use pyo3::{pyfunction, wrap_pyfunction}; -fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult>> { +fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult>> { let py = item.py(); let values = item.getattr("config")?.getattr("values")?; let origin = values.call_method1("get", ("migrate_from",))?; @@ -22,44 +26,23 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult> return Ok(None); } let materialization = config(item, "materialized")?; - Ok(Some(native( - py, - RecordId::ModelMigrationRequest, - &[ - ("model", item.clone()), - ( - "discovery", - registry::class(py, RecordId::MigrationDiscovery)?.getattr("MANUAL")?, - ), - ("raw_origin", origin.call_method0("strip")?), - ( - "force", - PyBool::new( - py, - values - .call_method1("get", ("migrate_force",))? - .is(PyBool::new(py, true)), - ) - .to_owned() - .into_any(), - ), - ( - "identity_only", - PyBool::new( - py, - materialization.eq("table")? || materialization.eq("view")?, - ) - .to_owned() - .into_any(), - ), - ], - )?)) + Ok(Some(ModelMigrationRequest { + model: item.clone(), + discovery: registry::class(py, RecordId::MigrationDiscovery)?.getattr("MANUAL")?, + raw_origin: Some(origin.call_method0("strip")?), + origin_location: None, + origin_model: py.None().into_bound(py), + force: values + .call_method1("get", ("migrate_force",))? + .is(PyBool::new(py, true)), + identity_only: materialization.eq("table")? || materialization.eq("view")?, + })) } -#[pyfunction] -#[pyo3(signature = (*, scope))] -fn manual_migration_requests<'py>(scope: &Bound<'py, PyAny>) -> PyResult> { - let mut requests: Vec> = Vec::new(); +fn manual_migration_requests<'py>( + scope: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut requests: Vec> = Vec::new(); for key in scope.getattr("execution_order")?.try_iter()? { let key = key?; if !scope.getattr("selected_keys")?.contains(&key)? @@ -76,32 +59,7 @@ fn manual_migration_requests<'py>(scope: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - options: Option>, -) -> PyResult> { - super::model_entries::decide(&KeywordArguments::read( - py, - "decide_model_migration", - options, - ( - &[ - "runtime", - "request", - "origin", - "origin_model", - "snapshot", - "state", - "snapshots_config", - ], - &[], - ), - )?) + Ok(requests) } /// Model migrations of the selected scope: discovered and declared moves, decided against live state. @@ -109,34 +67,11 @@ pub(crate) fn plan_model_migrations<'py>( phase: &MigrationPhase<'py>, fingerprints: &Bound<'py, PyAny>, ) -> PyResult> { - let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); - let schemas = call_with( - ®istry::native(py, NativeAttr::MigrationProjectSchemas)?, - &[("runtime", runtime.clone())], - )?; - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ( - "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ( - "database", - call_with( - ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, - &[("runtime", runtime.clone())], - )?, - ), - ("known_fingerprints", snapshot.getattr("fingerprints")?), - ], - )?; + let schemas = super::planning_origin::project_schemas(&runtime)?; + let state = + super::state_inspection::inspection(&runtime, Some(&snapshot.getattr("fingerprints")?))?; let discovery = discover((phase, fingerprints), &state, &schemas)?; finish(phase, &state, &schemas, &discovery) } @@ -145,12 +80,11 @@ fn discover<'py>( (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let runtime = phase.runtime.clone(); - let py = runtime.py(); let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); - let mut declared_requests: Vec> = Vec::new(); + let mut declared_requests: Vec> = Vec::new(); for item in runtime_field(&runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? @@ -165,63 +99,45 @@ fn discover<'py>( &[("schemas", schemas.clone())], )?; } - let mut declarations: Vec> = Vec::new(); - for request in declared_requests { - let resolved = call_with( - ®istry::native(py, NativeAttr::MigrationResolveOrigin)?, - &[ - ("request", request.clone()), - ("runtime", runtime.clone()), - ("state", state.clone()), - ], - )?; - declarations.push(native( - py, - RecordId::ModelMigrationDeclaration, - &[ - ("model_name", request.getattr("model")?.getattr("name")?), - ("origin_location", resolved.get_item(0)?), - ("origin_model", resolved.get_item(1)?), - ], - )?); + let mut declarations: Vec> = Vec::new(); + for request in &declared_requests { + let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; + declarations.push(ModelMigrationDeclaration { + model_name: request.model.getattr("name")?, + origin_location, + origin_model, + }); } - call_with( - ®istry::native(py, NativeAttr::DiscoverModelMigrations)?, - &[ - ("runtime", runtime.clone()), - ("scope", scope.clone()), - ("snapshot", snapshot.clone()), - ("manual_requests", manual_migration_requests(&scope)?), - ("declarations", PyTuple::new(py, declarations)?.into_any()), - ("state", state.clone()), - ("project_schemas", schemas.clone()), - ("fingerprints", fingerprints.clone()), - ], - ) + discover_model_migrations(DiscoveryRequest { + manual_requests: manual_migration_requests(&scope)?, + runtime, + scope, + snapshot, + declarations, + state: state.clone(), + project_schemas: schemas.clone(), + fingerprints: fingerprints.clone(), + }) } fn finish<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, - discovery: &Bound<'py, PyAny>, + discovery: &ModelMigrationDiscovery<'py>, ) -> PyResult> { let runtime = phase.runtime.clone(); let py = runtime.py(); let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); - let mut resolved: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for request in discovery.getattr("requests")?.try_iter()? { - let request = request?; - let origin = call_with( - ®istry::native(py, NativeAttr::MigrationResolveOrigin)?, - &[ - ("request", request.clone()), - ("runtime", runtime.clone()), - ("state", state.clone()), - ], - )?; - resolved.push((request, origin.get_item(0)?, origin.get_item(1)?)); + let mut resolved: Vec<( + &ModelMigrationRequest<'py>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + )> = Vec::new(); + for request in &discovery.requests { + let (origin, origin_model) = resolve_origin(request, &runtime, state)?; + resolved.push((request, origin, origin_model)); } let mut entries: Vec> = Vec::new(); let handovers = PyDict::new(py); @@ -229,10 +145,7 @@ fn finish<'py>( let origin_schemas = PySet::empty(py)?; let mut origins: Vec> = Vec::new(); for (_, origin, _) in &resolved { - call_with( - ®istry::native(py, NativeAttr::MigrationRequireSameDatabase)?, - &[("state", state.clone()), ("location", origin.clone())], - )?; + require_same_database(state, origin)?; let schema = origin.getattr("schema")?; if !schema.is_none() { origin_schemas.add(schema)?; @@ -253,39 +166,32 @@ fn finish<'py>( } else { project_config.getattr("snapshots")? }; - let policy = call_with( - ®istry::native(py, NativeAttr::MigrationMissingOriginPolicy)?, - &[("runtime", runtime.clone())], - )?; + let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { - let item = request.getattr("model")?; + let item = request.model.clone(); let name = item.getattr("name")?; - let decided = native( - py, - NativeAttr::DecideModelMigration, - &[ - ("runtime", runtime.clone()), - ("request", request), - ("origin", origin), - ("origin_model", origin_model), - ("snapshot", snapshot.clone()), - ("state", state.clone()), - ("snapshots_config", snapshots_config.clone()), - ], - )?; + let decided = super::model_entries::decide(&MigrationDecisionRequest { + runtime: runtime.clone(), + request, + origin, + origin_model, + snapshot: snapshot.clone(), + state: state.clone(), + snapshots_config: snapshots_config.clone(), + })?; entries.push(replace( - &decided.get_item(0)?, + &decided.entry, &[("missing_origin_policy", policy.clone())], )?); - if decided.get_item(2)?.is_truthy()? { + if decided.applies { let destinations = discovery - .getattr("destination_fingerprints")? + .destination_fingerprints .call_method1("get", (&name, PyTuple::empty(py)))?; handovers.set_item( name, super::model_handover::equivalent( &item, - decided.get_item(1)?, + decided.inherited, &destinations, &snapshot.getattr("column_dialect")?, )?, @@ -296,9 +202,7 @@ fn finish<'py>( let planned = PyDict::new(py); let mut renamed: Vec> = Vec::new(); let mut warnings: Vec> = Vec::new(); - for warning in discovery.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } + warnings.extend(discovery.warnings.iter().cloned()); for entry in &entries { let name = entry.getattr("model_name")?; if let Some(handover) = handovers.get_item(&name)? { @@ -341,8 +245,3 @@ fn finish<'py>( warnings, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(manual_migration_requests, module)?)?; - module.add_function(wrap_pyfunction!(decide_model_migration, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index abac12f8cd..4d04495439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -69,3 +69,82 @@ pub(crate) struct OldNameViewPlanning<'py> { pub(crate) views: Bound<'py, PyAny>, pub(crate) warnings: Vec>, } + +/// One destination model whose data should come from an earlier relation. +#[derive(Debug, Clone)] +pub(crate) struct ModelMigrationRequest<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) discovery: Bound<'py, PyAny>, + pub(crate) raw_origin: Option>, + pub(crate) origin_location: Option>, + pub(crate) origin_model: Bound<'py, PyAny>, + pub(crate) force: bool, + pub(crate) identity_only: bool, +} + +/// One project model's explicit migrate_from, resolved to a relation. +#[derive(Debug)] +pub(crate) struct ModelMigrationDeclaration<'py> { + pub(crate) model_name: Bound<'py, PyAny>, + pub(crate) origin_location: Bound<'py, PyAny>, + pub(crate) origin_model: Bound<'py, PyAny>, +} + +/// Manual and automatically discovered migration requests for one plan. +#[derive(Debug)] +pub(crate) struct ModelMigrationDiscovery<'py> { + pub(crate) requests: Vec>, + pub(crate) warnings: Vec>, + pub(crate) destination_fingerprints: Bound<'py, PyDict>, +} + +/// Compatibility status plus human-readable blocking findings. +#[derive(Debug)] +pub(crate) struct MigrationCompatibilityResult<'py> { + pub(crate) status: Bound<'py, PyAny>, + pub(crate) findings: Bound<'py, PyAny>, +} + +/// The fingerprint inputs shared by migration matching and destination fingerprints. +#[derive(Debug)] +pub(crate) struct FingerprintInputs<'py> { + pub(crate) metadata_jsons: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, PyAny>, + pub(crate) fingerprints: Bound<'py, PyAny>, +} + +/// One resolved migration request decided against live state. +#[derive(Debug)] +pub(crate) struct MigrationDecisionRequest<'a, 'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) request: &'a ModelMigrationRequest<'py>, + pub(crate) origin: Bound<'py, PyAny>, + pub(crate) origin_model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) state: Bound<'py, PyAny>, + pub(crate) snapshots_config: Bound<'py, PyAny>, +} + +/// What checking an origin relation's compatibility with its destination model reads. +#[derive(Debug)] +pub(crate) struct CompatibilityCheck<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) origin_relation: Bound<'py, PyAny>, + pub(crate) origin_columns: Bound<'py, PyAny>, + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) column_dialect: Bound<'py, PyAny>, + pub(crate) snapshots_config: Bound<'py, PyAny>, +} + +/// What discovering automatic and manual migration requests reads. +#[derive(Debug)] +pub(crate) struct DiscoveryRequest<'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) manual_requests: Vec>, + pub(crate) declarations: Vec>, + pub(crate) state: Bound<'py, PyAny>, + pub(crate) project_schemas: Bound<'py, PyAny>, + pub(crate) fingerprints: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index ff0045eecc..505092d9f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -8,7 +8,6 @@ use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldN use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; @@ -52,11 +51,7 @@ fn schemas<'py>( } } if include_project { - let project = native( - py, - NativeAttr::MigrationProjectSchemas, - &[("runtime", runtime)], - )?; + let project = super::planning_origin::project_schemas(&runtime)?; for schema in project.try_iter()? { needed.add(schema?)?; } @@ -95,27 +90,7 @@ pub(crate) fn plan_old_name_views<'py>( selected.push(item); } } - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ( - "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ( - "database", - native( - py, - NativeAttr::MigrationPlanningDatabase, - &[("runtime", runtime.clone())], - )?, - ), - ], - )?; + let state = super::state_inspection::inspection(&runtime, None)?; let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index 7afa215fba..a3398a3f48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -1,53 +1,34 @@ //! Assemble migration policy and redirected live cursor inspection requests. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::retention::effective_target_config; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -#[pyfunction] -#[pyo3(signature = (*, runtime))] -fn migration_missing_origin_policy<'py>( +/// The target's policy for migration origins that no longer exist. +pub(super) fn missing_origin_policy<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - ®istry::native(runtime.py(), NativeAttr::EffectiveTargetConfig)?, - &[("runtime", runtime.clone())], - )? - .getattr("missing_migration_origin") + effective_target_config(runtime)?.getattr("missing_migration_origin") } -#[pyfunction] -#[pyo3(signature = (**options))] -fn migration_redirected_cursor_snapshots<'py>( - py: Python<'py>, - options: Option>, +/// Live cursor snapshots of models whose history now sits in a redirected relation. +pub(super) fn redirected_cursor_snapshots<'py>( + phase: &MigrationPhase<'py>, + (existing_relations, target_relations): (Bound<'py, PyAny>, Bound<'py, PyAny>), + origin_cursor_columns: Option>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "migration_redirected_cursor_snapshots", - options, - ( - &[ - "runtime", - "scope", - "existing_relations", - "target_relations", - "overrides", - "deferral", - ], - &["origin_cursor_columns"], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; - let overrides = args.value("overrides")?; - let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; + let runtime = &phase.runtime; + let py = runtime.py(); + let overrides = &phase.overrides; + let scope = &phase.scope; + let project = runtime_field(runtime, |runtime| &runtime.project)?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let start = if config.is_none() { py.None().into_bound(py) } else { @@ -61,18 +42,15 @@ fn migration_redirected_cursor_snapshots<'py>( ("runtime_producer_keys", scope.getattr("selected_keys")?), ( "invocation_time", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + runtime_field(runtime, |runtime| &runtime.invocation_time)?, ), ("start_cursor_config", start), ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], )?; - let full_refresh = call_with( - ®istry::native(py, NativeAttr::EffectivelyFullRefreshedModelNames)?, - &[ - ("project", project.clone()), - ("cli_full_refresh", overrides.getattr("full_refresh")?), - ], + let full_refresh = effectively_full_refreshed_model_names( + &project, + overrides.getattr("full_refresh")?.is_truthy()?, )?; call_with( ®istry::native(py, NativeAttr::GatherRedirectedCursorSnapshots)?, @@ -80,36 +58,28 @@ fn migration_redirected_cursor_snapshots<'py>( ("project", project), ( "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, + runtime_field(runtime, |runtime| &runtime.adapter)?, ), ( "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, + runtime_field(runtime, |runtime| &runtime.connection)?, ), - ("existing_relations", args.value("existing_relations")?), - ("target_relations", args.value("target_relations")?), + ("existing_relations", existing_relations), + ("target_relations", target_relations), ("cursor_scope", cursor_scope), ("full_refresh_model_names", full_refresh), ( "deferred_locations", - args.value("deferral")?.getattr("deferred_locations")?, + phase.deferral.getattr("deferred_locations")?, ), ( "on_progress", - runtime_field(&runtime, |runtime| &runtime.on_progress)?, + runtime_field(runtime, |runtime| &runtime.on_progress)?, ), ( "origin_cursor_columns", - args.value("origin_cursor_columns")?, + origin_cursor_columns.unwrap_or_else(|| py.None().into_bound(py)), ), ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_missing_origin_policy, module)?)?; - module.add_function(wrap_pyfunction!( - migration_redirected_cursor_snapshots, - module - )?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs index 2fd4af82e1..97a010b63c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -7,9 +7,8 @@ use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; -use pyo3::{pyfunction, wrap_pyfunction}; fn label<'py>(location: &Bound<'py, PyAny>) -> PyResult> { let qualified = location.getattr("qualified_name")?; @@ -20,8 +19,6 @@ fn label<'py>(location: &Bound<'py, PyAny>) -> PyResult> { } } -#[pyfunction] -#[pyo3(signature = (*, runtime, decision))] fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, @@ -40,9 +37,8 @@ fn migration_promotion<'py>( promotion.getattr("RENAME") } -#[pyfunction] -#[pyo3(signature = (*, runtime, model, entry))] -fn migration_with_execution<'py>( +/// The migration entry with the adapter's staging, transfer and promotion choices. +pub(super) fn migration_with_execution<'py>( runtime: &Bound<'py, PyAny>, model: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, @@ -91,8 +87,3 @@ fn migration_with_execution<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_promotion, module)?)?; - module.add_function(wrap_pyfunction!(migration_with_execution, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 0f3289d8a2..de08a94111 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -2,12 +2,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PySet, PySetMethods, PyTuple}; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use pyo3::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods}; pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, @@ -32,17 +33,14 @@ fn named<'py>( destination: &Bound<'py, PyAny>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { let item = item?; if item.getattr("name")?.eq(name)? { - return Ok( - PyTuple::new(runtime.py(), [item.getattr("destination")?, name.clone()])? - .into_any(), - ); + return Ok((item.getattr("destination")?, name.clone())); } } let fingerprint = call_with( @@ -62,7 +60,7 @@ fn named<'py>( ("name", name.clone()), ] }; - Ok(PyTuple::new(runtime.py(), [location(runtime, &fields)?, name.clone()])?.into_any()) + Ok((location(runtime, &fields)?, name.clone())) } fn input_error(py: Python<'_>, message: String) -> PyResult { @@ -71,9 +69,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { )) } -#[pyfunction] -#[pyo3(signature = (*, runtime))] -fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { +/// The database migration planning inspects: the first model destination's, else the target's. +pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = project.getattr("effective_target_database")?; for item in project.getattr("models")?.try_iter()? { @@ -85,9 +82,8 @@ fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { +/// The schemas the project's models write to. +pub(super) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? @@ -101,9 +97,8 @@ fn migration_project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult, location: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -133,25 +128,21 @@ fn migration_require_same_database( )?) } -#[pyfunction] -#[pyo3(signature = (*, request, runtime, state))] -fn migration_resolve_origin<'py>( - request: &Bound<'py, PyAny>, +/// The origin location and origin model name of one migration request. +pub(super) fn resolve_origin<'py>( + request: &ModelMigrationRequest<'py>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = runtime.py(); - let explicit = request.getattr("origin_location")?; - if !explicit.is_none() { - return Ok(PyTuple::new(py, [explicit, request.getattr("origin_model")?])?.into_any()); + if let Some(explicit) = &request.origin_location { + return Ok((explicit.clone(), request.origin_model.clone())); } - let raw = request.getattr("raw_origin")?; - let raw = if raw.is_truthy()? { - raw - } else { - super::column_objects::text(py, "") + let raw = match &request.raw_origin { + Some(raw) if raw.is_truthy()? => raw.clone(), + _ => super::column_objects::text(py, ""), }; - let item = request.getattr("model")?; + let item = request.model.clone(); let destination = item.getattr("destination")?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { @@ -191,12 +182,5 @@ fn migration_resolve_origin<'py>( ("name", parts[parts.len() - 1].clone()), ], )?; - Ok(PyTuple::new(py, [origin, py.None().into_bound(py)])?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_planning_database, module)?)?; - module.add_function(wrap_pyfunction!(migration_project_schemas, module)?)?; - module.add_function(wrap_pyfunction!(migration_require_same_database, module)?)?; - module.add_function(wrap_pyfunction!(migration_resolve_origin, module)?) + Ok((origin, py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 576f52c48e..c0f4e37068 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -3,16 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::discovery::register(module)?; super::compatibility::register(module)?; - super::matching::register(module)?; super::old_names::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; - super::model_planning::register(module)?; - super::planning_boundary::register(module)?; - super::planning_origin::register(module)?; - super::planning_execution::register(module)?; super::projections::register(module)?; module.add_class::()?; super::fingerprint::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index ad6d16ca7e..77f3b74721 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::constants::{ GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; @@ -578,3 +579,17 @@ impl MigrationStateInspection { PyTuple::new(py, orphans) } } + +/// A fresh live-state inspection over the runtime's connection and planning database. +pub(super) fn inspection<'py>( + runtime: &Bound<'py, PyAny>, + known_fingerprints: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let state = MigrationStateInspection::new( + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, + &super::planning_origin::planning_database(runtime)?, + known_fingerprints, + )?; + Ok(Bound::new(runtime.py(), state)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 38d34abd6b..9cbc14ad68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1335,45 +1335,6 @@ const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { FieldSpec::new("annotation_module", FieldDefault::None), ], }; -const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { - name: "ModelMigrationRequest", - doc: "One destination model whose data should come from an earlier relation.", - fields: &[ - FieldSpec::required("model"), - FieldSpec::required("discovery"), - FieldSpec::new("raw_origin", FieldDefault::None), - FieldSpec::new("origin_location", FieldDefault::None), - FieldSpec::new("origin_model", FieldDefault::None), - FieldSpec::new("force", FieldDefault::False), - FieldSpec::new("identity_only", FieldDefault::False), - ], -}; -const R_MODEL_MIGRATION_DECLARATION: RecordSpec = RecordSpec { - name: "ModelMigrationDeclaration", - doc: "One project model's explicit migrate_from, resolved to a relation.", - fields: &[ - FieldSpec::required("model_name"), - FieldSpec::required("origin_location"), - FieldSpec::required("origin_model"), - ], -}; -const R_MODEL_MIGRATION_DISCOVERY: RecordSpec = RecordSpec { - name: "ModelMigrationDiscovery", - doc: "Manual and automatically discovered migration requests for one plan.", - fields: &[ - FieldSpec::new("requests", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - FieldSpec::new("destination_fingerprints", FieldDefault::EmptyDict), - ], -}; -const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { - name: "MigrationCompatibilityResult", - doc: "Compatibility status plus human-readable blocking findings.", - fields: &[ - FieldSpec::required("status"), - FieldSpec::new("findings", FieldDefault::EmptyTuple), - ], -}; /// Planner value records, in registration order. pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1412,10 +1373,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_SCENARIO_ARTIFACT_IDENTITY, R_SCENARIO_ARTIFACT_NAME, R_PLAN_PROVIDER_USAGE, - R_MODEL_MIGRATION_REQUEST, - R_MODEL_MIGRATION_DECLARATION, - R_MODEL_MIGRATION_DISCOVERY, - R_MIGRATION_COMPATIBILITY_RESULT, ]; /// Planner string enums, in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 61cf4bd37d..c6574dadcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -320,7 +320,6 @@ native_ids! { MicrobatchStrategy => "MicrobatchStrategy", MicrobatchesConfig => "MicrobatchesConfig", MigrationCompatibility => "MigrationCompatibility", - MigrationCompatibilityResult => "MigrationCompatibilityResult", MigrationDecision => "MigrationDecision", MigrationDiscovery => "MigrationDiscovery", MigrationEvent => "MigrationEvent", @@ -333,10 +332,7 @@ native_ids! { ModelColumnLineage => "ModelColumnLineage", ModelCursorSnapshot => "ModelCursorSnapshot", ModelInputBuildContext => "ModelInputBuildContext", - ModelMigrationDeclaration => "ModelMigrationDeclaration", - ModelMigrationDiscovery => "ModelMigrationDiscovery", ModelMigrationPlanEntry => "ModelMigrationPlanEntry", - ModelMigrationRequest => "ModelMigrationRequest", ModelPlanEntry => "ModelPlanEntry", ModelSchemaDeclaration => "ModelSchemaDeclaration", ModelSql => "ModelSql", @@ -761,7 +757,6 @@ native_ids! { CheckCompileReuse => "check_compile_reuse", CheckCursorTypeConsistency => "check_cursor_type_consistency", CheckFullRebuildAllowed => "check_full_rebuild_allowed", - CheckMigrationCompatibility => "check_migration_compatibility", CheckSelectedScopeBuildability => "check_selected_scope_buildability", ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", @@ -807,7 +802,6 @@ native_ids! { DbtInitProgressReporter => "DbtInitProgressReporter", DbtRunner => "DbtRunner", DebugDbtFromProject => "debug_dbt_from_project", - DecideModelMigration => "decide_model_migration", DeclarationRows => "declaration_rows", DeclaredColumnAliases => "declared_column_aliases", DeclaredRelations => "declared_relations", @@ -822,7 +816,6 @@ native_ids! { DiscoverHookFunctions => "discover_hook_functions", DiscoverMaterializationFiles => "discover_materialization_files", DiscoverModelFiles => "discover_model_files", - DiscoverModelMigrations => "discover_model_migrations", DiscoverModelSchemaFiles => "discover_model_schema_files", DiscoverProjectInputs => "discover_project_inputs", DiscoverProviderClasses => "discover_provider_classes", @@ -928,24 +921,14 @@ native_ids! { MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", - MatchMigrationFingerprints => "match_migration_fingerprints", MatchProjectRelation => "match_project_relation", MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", MergeRecomputedModelChanges => "merge_recomputed_model_changes", MicrobatchGuidance => "microbatch_guidance", - MigrationDestinationFingerprints => "migration_destination_fingerprints", MigrationFingerprintCache => "MigrationFingerprintCache", MigrationFingerprintRefNames => "migration_fingerprint_ref_names", - MigrationMissingOriginPolicy => "migration_missing_origin_policy", - MigrationPlanningDatabase => "migration_planning_database", - MigrationProjectSchemas => "migration_project_schemas", - MigrationRedirectedCursorSnapshots => "migration_redirected_cursor_snapshots", - MigrationRequireSameDatabase => "migration_require_same_database", - MigrationResolveOrigin => "migration_resolve_origin", - MigrationStateInspection => "MigrationStateInspection", - MigrationWithExecution => "migration_with_execution", MissingUpstreamMessage => "missing_upstream_message", ModelDefinitionHash => "model_definition_hash", ModelHeaderFailure => "model_header_failure", @@ -1153,7 +1136,6 @@ native_ids! { StandalonePivotModels => "standalone_pivot_models", StartModelAnalysisSession => "start_model_analysis_session", StaticSqlTestPlanningDiagnostics => "static_sql_test_planning_diagnostics", - StoredMigrationFingerprint => "stored_migration_fingerprint", StoredOldNameViewColumns => "stored_old_name_view_columns", TerminalEventIndex => "TerminalEventIndex", TerminalEventIndexContext => "_terminal_event_index_context", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 34b4a4c8dc..3d44ad436d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -256,32 +256,6 @@ def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[ STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] -def migration_planning_database(*, runtime: PlannerRuntime) -> str | None: ... -def migration_project_schemas(*, runtime: PlannerRuntime) -> set[str]: ... -def migration_missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: ... -def migration_require_same_database( - *, state: MigrationStateInspection, location: CompiledRelationLocation -) -> None: ... -def migration_resolve_origin( - *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection -) -> tuple[CompiledRelationLocation, str | None]: ... -def migration_promotion( - *, runtime: PlannerRuntime, decision: MigrationDecision -) -> MigrationPromotion: ... -def migration_with_execution( - *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry -) -> ModelMigrationPlanEntry: ... -def migration_redirected_cursor_snapshots( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - existing_relations: dict[str, RelationInfo], - target_relations: dict[str, str], - overrides: PlannerOverrides, - deferral: DeferralInputs, - origin_cursor_columns: dict[str, str] | None = None, -) -> dict[str, ModelCursorSnapshot]: ... - class CompileInputReads: """Environment, settings, and run-identity reads observed during one compile.""" @@ -1380,61 +1354,6 @@ class PlannerWarehouseState: migration_fingerprints: MigrationFingerprintCache = ..., ) -> None: ... -class ModelMigrationRequest(NativeRecord): - """One destination model whose data should come from an earlier relation.""" - - model: CompiledModelView - discovery: MigrationDiscovery - raw_origin: str | None - origin_location: CompiledRelationLocation | None - origin_model: str | None - force: bool - identity_only: bool - def __init__( - self, - model: CompiledModelView, - discovery: MigrationDiscovery, - raw_origin: str | None = ..., - origin_location: CompiledRelationLocation | None = ..., - origin_model: str | None = ..., - force: bool = ..., - identity_only: bool = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationRequest: ... - -class ModelMigrationDeclaration(NativeRecord): - """One project model's explicit migrate_from, resolved to a relation.""" - - model_name: str - origin_location: CompiledRelationLocation - origin_model: str | None - def __init__( - self, model_name: str, origin_location: CompiledRelationLocation, origin_model: str | None - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationDeclaration: ... - -class ModelMigrationDiscovery(NativeRecord): - """Manual and automatically discovered migration requests for one plan.""" - - requests: tuple[ModelMigrationRequest, ...] - warnings: tuple[PlanWarning, ...] - destination_fingerprints: dict[str, tuple[str, ...]] - def __init__( - self, - requests: tuple[ModelMigrationRequest, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - destination_fingerprints: dict[str, tuple[str, ...]] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationDiscovery: ... - -class MigrationCompatibilityResult(NativeRecord): - """Compatibility status plus human-readable blocking findings.""" - - status: MigrationCompatibility - findings: tuple[str, ...] - def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... - def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... - class PlannerIdentityContext: """Expected version identities for inspection and stale-warning scopes.""" @@ -7027,15 +6946,6 @@ def build_migration_fingerprint( *, query_sql: str, metadata_json: str, ref_identities: Mapping[str, str], dialect: str | None ) -> str | None: ... def migration_fingerprint_ref_names(*, query_sql: str, metadata_json: str) -> frozenset[str]: ... -def check_migration_compatibility( - *, - model: CompiledModelView, - origin_relation: RelationInfo, - origin_columns: tuple[ColumnInfo, ...], - sql_analysis_enabled: bool, - column_dialect: str | None, - snapshots_config: SnapshotsConfig, -) -> MigrationCompatibilityResult: ... def migration_incremental_blocking_findings( *, model: CompiledModelView, @@ -7049,48 +6959,7 @@ def migration_snapshot_blocking_findings( dialect: str | None, snapshots_config: SnapshotsConfig, ) -> tuple[str, ...]: ... -def match_migration_fingerprints( - *, - models: tuple[CompiledModelView, ...], - new_names: frozenset[str], - seed_variants: tuple[dict[str, str], ...], - candidates: tuple[Fingerprint, ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: ... -def migration_destination_fingerprints( - *, - requests: tuple[ModelMigrationRequest, ...], - variants: tuple[dict[str, str], ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> dict[str, tuple[str, ...]]: ... -def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... -def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... -def discover_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - manual_requests: tuple[ModelMigrationRequest, ...], - declarations: tuple[ModelMigrationDeclaration, ...], - state: MigrationStateInspection, - project_schemas: set[str], - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationDiscovery: ... -def decide_model_migration( - *, - runtime: PlannerRuntime, - request: ModelMigrationRequest, - origin: CompiledRelationLocation, - origin_model: str | None, - snapshot: WarehouseSnapshot, - state: MigrationStateInspection, - snapshots_config: SnapshotsConfig, -) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: ... def plan_column_migrations( *, runtime: PlannerRuntime, From a417acac2831209f08f07339e84f91cc471b6152 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:28:36 +0100 Subject: [PATCH 3/4] refactor(planner): call native planner helpers directly Planner resolution, entry, migration, scenario, warehouse, audit and command-plan code calls the native helpers it needs as Rust functions instead of looking them up on the native module and calling them with keyword dictionaries. Keyword-only helpers receive their arguments directly; signature helpers receive typed values. SQL code scanning in reference resolution uses the scanner directly, and the migration fingerprint cache default is constructed natively. Calls that tests deliberately replace on the native module stay module lookups: SQL test planning, migration fingerprint computation, table type planning, and the compile pipeline's execution plan and Python run outputs. The stale-identity unit test now supplies the project fields change detection reads instead of replacing it. --- .../_helpers/analysis/explicit_references.rs | 4 +- .../src/bindings/_helpers/auditing/mod.rs | 2 +- .../src/bindings/_helpers/auditing/render.rs | 2 +- .../_helpers/authored_values/project_vars.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 139 +++++++------ .../src/bindings/_helpers/check/check_core.rs | 34 ++-- .../src/bindings/_helpers/cli/progress.rs | 2 +- .../bindings/_helpers/cli/progress_text.rs | 5 +- .../_helpers/command_runtime/connection.rs | 2 +- .../command_runtime/preview_target.rs | 2 +- .../_helpers/compile_driver/deferral.rs | 6 +- .../_helpers/compile_driver/discovery.rs | 2 +- .../compile_driver/pipeline_entries.rs | 4 +- .../_helpers/compile_driver/plan_result.rs | 109 ++++++---- .../_helpers/compile_driver/project.rs | 4 +- .../compile_driver/python_plan_entries.rs | 12 +- .../compile_driver/relation_targets.rs | 4 +- .../compile_driver/selection_phases.rs | 2 +- .../_helpers/contracts/spec_resolution.rs | 2 +- .../src/bindings/_helpers/cursor/intervals.rs | 2 +- .../src/bindings/_helpers/cursor/scalars.rs | 4 +- .../bindings/_helpers/discovery/guidance.rs | 2 +- .../bindings/_helpers/discovery/locations.rs | 2 +- .../_helpers/fingerprints/latest_rows.rs | 2 +- .../src/bindings/_helpers/fingerprints/mod.rs | 4 +- .../_helpers/fingerprints/query_hashes.rs | 6 +- .../_helpers/fingerprints/storage_io.rs | 2 +- .../bindings/_helpers/freshness/state_io.rs | 2 +- .../src/bindings/_helpers/graph/algorithms.rs | 2 +- .../_helpers/graph/execution_gates.rs | 5 +- .../src/bindings/_helpers/graph/mod.rs | 4 +- .../_helpers/graph/python_lifecycle.rs | 2 +- .../_helpers/graph/python_selection.rs | 2 +- .../_helpers/janitor/janitor_phases.rs | 27 ++- .../bindings/_helpers/plan_output/display.rs | 2 +- .../_helpers/planner_changes/detect.rs | 19 +- .../_helpers/planner_changes/metadata.rs | 2 +- .../_helpers/planner_changes/renames.rs | 18 +- .../_helpers/planner_entries/clone_plans.rs | 126 ++++++------ .../_helpers/planner_entries/command_plans.rs | 87 ++++---- .../_helpers/planner_entries/display_plan.rs | 28 ++- .../planner_entries/execution_plan.rs | 2 +- .../bindings/_helpers/planner_entries/mod.rs | 4 +- .../_helpers/planner_entries/model_entries.rs | 80 ++++---- .../_helpers/planner_entries/plan_output.rs | 107 +++++----- .../_helpers/planner_entries/relations.rs | 130 ++++++------ .../_helpers/planner_identity/graph.rs | 18 +- .../_helpers/planner_identity/hashing.rs | 2 +- .../_helpers/planner_identity/identities.rs | 6 +- .../_helpers/planner_identity/metadata.rs | 10 +- .../_helpers/planner_identity/staleness.rs | 2 +- .../planner_migrations/column_candidates.rs | 22 +- .../planner_migrations/column_entries.rs | 62 +++--- .../planner_migrations/column_objects.rs | 21 +- .../planner_migrations/column_overlay.rs | 26 ++- .../_helpers/planner_migrations/columns.rs | 4 +- .../planner_migrations/compatibility.rs | 37 ++-- .../planner_migrations/discovery_models.rs | 35 ++-- .../planner_migrations/fingerprint.rs | 6 +- .../_helpers/planner_migrations/mod.rs | 6 +- .../planner_migrations/model_handover.rs | 17 +- .../planner_migrations/model_planning.rs | 32 +-- .../planner_migrations/old_name_entries.rs | 39 ++-- .../planner_migrations/old_name_planning.rs | 38 ++-- .../planner_migrations/old_name_references.rs | 41 ++-- .../_helpers/planner_migrations/old_names.rs | 8 +- .../planner_migrations/planning_boundary.rs | 67 +++--- .../planner_migrations/projections.rs | 7 +- .../planner_migrations/state_inspection.rs | 10 +- .../_helpers/planner_phases/buildability.rs | 61 +++--- .../planner_phases/execution_changes.rs | 42 ++-- .../_helpers/planner_phases/objects.rs | 10 + .../planner_phases/output_assembly.rs | 130 +++++++----- .../_helpers/planner_phases/retention.rs | 13 +- .../_helpers/planner_phases/runtime.rs | 9 +- .../_helpers/planner_phases/scopes.rs | 79 ++++---- .../planner_resolution/cursor_bounds.rs | 18 +- .../planner_resolution/cursor_model.rs | 159 ++++++++------- .../planner_resolution/cursor_policies.rs | 2 +- .../planner_resolution/cursor_ranges.rs | 160 +++++++-------- .../planner_resolution/cursor_windows.rs | 14 +- .../planner_resolution/future_safety.rs | 2 +- .../_helpers/planner_resolution/locations.rs | 10 +- .../planner_resolution/maximum_safety.rs | 21 +- .../_helpers/planner_resolution/mod.rs | 14 +- .../planner_resolution/orchestration.rs | 2 +- .../_helpers/planner_resolution/refs.rs | 22 +- .../_helpers/planner_resolution/sources.rs | 2 +- .../bindings/_helpers/planner_scenario/cli.rs | 87 ++++---- .../_helpers/planner_scenario/entries.rs | 60 +++--- .../_helpers/planner_scenario/execution.rs | 70 ++++--- .../bindings/_helpers/planner_scenario/mod.rs | 2 +- .../bindings/_helpers/planner_scenario/sql.rs | 8 +- .../bindings/_helpers/planner_scope/mod.rs | 6 +- .../_helpers/planner_selection/selectors.rs | 10 +- .../_helpers/planner_selection/unit_tests.rs | 2 +- .../planner_strategy/audit_entries.rs | 39 ++-- .../_helpers/planner_strategy/cursor_types.rs | 2 +- .../_helpers/planner_warehouse/constants.rs | 6 +- .../_helpers/planner_warehouse/cursors.rs | 191 +++++++----------- .../planner_warehouse/fingerprint_cache.rs | 22 +- .../_helpers/planner_warehouse/snapshot.rs | 49 ++--- .../planner_warehouse/source_deferral.rs | 32 ++- .../planner_warehouse/source_tables.rs | 35 ++-- .../_helpers/planner_warehouse/watermarks.rs | 10 +- .../src/bindings/_helpers/references/mod.rs | 2 +- .../bindings/_helpers/references/relations.rs | 2 +- .../_helpers/result_output/documents.rs | 2 +- .../_helpers/result_output/terminal_index.rs | 2 +- .../semantic_shapes/source_bindings.rs | 2 +- .../_helpers/sql_test_glue/planning.rs | 8 +- .../bindings/_helpers/sqltext/code_scanner.rs | 6 +- .../_helpers/state_rows/migration_io.rs | 6 +- .../_helpers/planning/test_identities.py | 7 +- 114 files changed, 1494 insertions(+), 1472 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 47d9c379c9..f7b8cbfe1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -386,7 +386,7 @@ fn python_literal_sql_calls( /// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. #[pyfunction] #[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] -fn hard_coded_relation_remedy( +pub(crate) fn hard_coded_relation_remedy( owner_kind: &str, r#ref: &Bound<'_, PyAny>, upstream_loader_by_source: Option<&Bound<'_, PyAny>>, @@ -407,7 +407,7 @@ fn hard_coded_relation_remedy( /// The P008 message for SQL naming a relation kept only as a compatibility view. #[pyfunction] #[pyo3(signature = (*, owner_label, written, method, model_name))] -fn old_name_reference_message( +pub(crate) fn old_name_reference_message( owner_label: &str, written: &str, method: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs index 06800fcfe3..5b11d2bbb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs @@ -1,5 +1,5 @@ mod identity; mod measurement; pub(crate) mod registration; -mod render; +pub(crate) mod render; mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs index a40db46208..3e07e6b131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs @@ -30,7 +30,7 @@ fn qualified<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>) -> PyResult { +pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("render_audit_sql", request)?; let sql: String = request.get("unresolved_sql")?.extract()?; let models: Bound<'_, PyAny> = request.get("model_locations")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index 727dc3ce7e..7d3c5dc290 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -45,7 +45,7 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { /// One project var value as interpolated text; structured values are rejected. #[pyfunction] #[pyo3(signature = (*, value, label))] -fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { +pub(crate) fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { project_var_text(value, label) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f23c9beec0..e3a194e6f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -1,5 +1,17 @@ //! `sqb check`: run the selected Python checks after their Python dependencies. +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::compile_driver::selection_phases::compile_static_command_context; +use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; +use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; +use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; +use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -163,16 +175,14 @@ fn compile_pipeline<'py>( &graph.getattr("selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; - let closure: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::TransitiveClosureMany)?, - &[ - ("starts", check_names.clone().into_any()), - ("edges", graph.getattr("upstream_deps")?), - ( - "include_starts", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + let closure: Bound<'py, PyAny> = object( + py, + transitive_closure_many( + check_names.as_any(), + &graph.getattr("upstream_deps")?, + false, + None, + )?, )?; let loader_kind: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?.getattr("LOADER")?; @@ -207,12 +217,9 @@ fn compile_pipeline<'py>( } } let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); - let resolver: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveExternalSqlReferenceResolver)?, - &[ - ("project_dir", invocation.getattr("effective_project_dir")?), - ("discovered_inputs", discovered.clone()), - ], + let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver( + invocation.getattr("effective_project_dir")?, + discovered.clone(), )?; let options: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompilePipelineOptions)?, @@ -227,8 +234,8 @@ fn compile_pipeline<'py>( ("external_sql_reference_resolver", resolver), ], )?; - let context: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileStaticCommandContext)?, + let context: Bound<'py, PyAny> = compile_static_command_context(Some(&keywords( + py, &[ ("discovered_inputs", discovered), ("adapter", invocation.getattr("adapter")?), @@ -242,18 +249,18 @@ fn compile_pipeline<'py>( .getattr("on_progress")?, ), ], + )?))?; + let plan_output: Bound<'py, PyAny> = build_relation_command_plan( + &context.getattr("project")?, + &context.getattr("scope")?, + &context.getattr("relations")?, )?; - let plan_output: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildRelationCommandPlan)?, - &[ - ("project", context.getattr("project")?), - ("scope", context.getattr("scope")?), - ("relations", context.getattr("relations")?), - ], - )?; - call_with( - ®istry::native(py, NativeAttr::BuildStaticPipelineResult)?, - &[("context", context), ("plan_output", plan_output)], + build_static_pipeline_result( + py, + Some(keywords( + py, + &[("context", context), ("plan_output", plan_output)], + )?), ) } @@ -275,15 +282,12 @@ fn python_selection<'py>( request: &Bound<'py, PyAny>, discovered: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { - let py: Python<'py> = request.py(); + let _py: Python<'py> = request.py(); let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; - call_with( - ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, - &[ - ("select", request.getattr("select")?), - ("exclude", request.getattr("exclude")?), - ("discovered_inputs", discovered.clone()), - ], + reject_unit_test_selectors( + &request.getattr("select")?, + &request.getattr("exclude")?, + discovered, )?; let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; Ok((python_graph, check_names)) @@ -314,27 +318,28 @@ fn prepare<'py>( ("python_node_names", PyFrozenSet::empty(py)?.into_any()), ], )?; - let lifecycle_plan: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonSqlRunLifecycle)?, - &[ - ("selection", selection), - ("python_graph", python_graph.clone()), - ], - )?; + let lifecycle_plan: Bound<'py, PyAny> = + build_python_sql_run_lifecycle(&selection, &python_graph)?; let relation_refs: Bound<'py, PyAny> = call_with( &python_graph.getattr("selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; let project: Bound<'py, PyAny> = pipeline.getattr("project")?; let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; - let relation_targets: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonRelationTargets)?, - &[ - ("adapter", adapter.clone()), - ("project", project.clone()), - ("plan_output", plan_output.clone()), - ("required_refs", relation_refs.clone()), - ], + let relation_targets: Bound<'py, PyAny> = object( + py, + build_python_relation_targets( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("project", project.clone()), + ("plan_output", plan_output.clone()), + ("required_refs", relation_refs.clone()), + ], + )?), + )?, )?; Ok(Prepared { check_functions: PyTuple::new(py, check_functions)?.into_any(), @@ -353,9 +358,15 @@ fn prepare<'py>( ("effective_target_schema", "default_schema"), )?, ), - project_relations: call_with( - ®istry::native(py, NativeAttr::BuildProjectRelationTargets)?, - &[("adapter", adapter), ("plan_output", plan_output)], + project_relations: object( + py, + build_project_relation_targets( + py, + Some(keywords( + py, + &[("adapter", adapter), ("plan_output", plan_output)], + )?), + )?, )?, python_graph, pipeline, @@ -617,16 +628,14 @@ fn complete<'py>( .call_method1("__truediv__", ("target",))?, &tuple, )?; - call_with( - ®istry::native(py, NativeAttr::WriteExecutionJsonOutput)?, - &[ - ( - "payload", - PyString::new(py, &results_json(results)?).into_any(), - ), - ("json_output", request.getattr("json_output")?), - ("json_output_path", request.getattr("json_output_path")?), - ], + object( + py, + write_execution_json_output( + py, + PyString::new(py, &results_json(results)?).into_any(), + request.getattr("json_output")?.is_truthy()?, + request.getattr("json_output_path")?, + )?, )?; Ok(check_exit_code(&texts)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 44a352eb20..0937a8a3f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -1,5 +1,9 @@ //! Python check planning: run-state recording, dependency relations and read-side preflight. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::graph::python_selection::resolve_python_check_excludes; +use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -15,8 +19,6 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const MISSING_RELATION_CODE: &str = "C682"; const MODEL_KIND: &str = "model"; @@ -170,14 +172,20 @@ pub(crate) fn build_check_relation_targets<'py>( &python_graph.getattr("selected_sql_refs")?, &[("selected_names", selected_python_names)], )?; - call_with( - ®istry::native(py, NativeAttr::BuildPythonRelationTargets)?, - &[ - ("adapter", adapter.clone()), - ("project", pipeline_result.getattr("project")?), - ("plan_output", pipeline_result.getattr("plan_output")?), - ("required_refs", required), - ], + object( + py, + build_python_relation_targets( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("project", pipeline_result.getattr("project")?), + ("plan_output", pipeline_result.getattr("plan_output")?), + ("required_refs", required), + ], + )?), + )?, ) } @@ -189,9 +197,9 @@ pub(crate) fn relevant_check_functions<'py>( (selected_dependency_names, completed_sql_names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult>> { let py: Python<'py> = discovered_inputs.py(); - let excluded: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolvePythonCheckExcludes)?, - &[("graph", python_graph.clone()), ("exclude", exclude)], + let excluded: Bound<'py, PyAny> = object( + py, + resolve_python_check_excludes(python_graph, exclude.extract::>()?)?, )?; let check_kind: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?.getattr("CHECK")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index bf1bb54efb..8236109929 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -314,7 +314,7 @@ fn context_variable(py: Python<'_>) -> PyResult> { /// Return the projector installed for the current CLI invocation. #[pyfunction] -fn current_native_progress_projector(py: Python<'_>) -> PyResult> { +pub(crate) fn current_native_progress_projector(py: Python<'_>) -> PyResult> { context_variable(py)?.call_method0("get") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index a5e428ab38..2792dbbead 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -35,7 +35,10 @@ fn format_execution_header(command: &str, target: Option<&str>, concurrency: i64 /// Write an execution header for command progress output. #[pyfunction] #[pyo3(signature = (**options))] -fn write_execution_header(py: Python<'_>, options: Option>) -> PyResult<()> { +pub(crate) fn write_execution_header( + py: Python<'_>, + options: Option>, +) -> PyResult<()> { let arguments: KeywordArguments<'_> = KeywordArguments::read(py, "write_execution_header", options, HEADER_OPTIONS)?; let command: String = arguments.value("command")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 2ccae8393e..d1ae0ad2a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -279,7 +279,7 @@ pub(crate) fn project_connection_config<'py>( /// Resolve the project connection; offline modes pass `require_connection=False`. #[pyfunction] #[pyo3(signature = (**options))] -fn resolve_project_connection_config<'py>( +pub(crate) fn resolve_project_connection_config<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs index 0bfa4096cf..a57cd9990e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs @@ -19,7 +19,7 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { /// Fail clearly when `--as` names a target that is not configured. #[pyfunction] #[pyo3(signature = (*, discovered_inputs, as_target, command_name))] -fn validate_preview_target( +pub(crate) fn validate_preview_target( py: Python<'_>, discovered_inputs: Bound<'_, PyAny>, as_target: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 3f5f2f5bd3..b7f9b6bf1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -29,7 +29,7 @@ struct DeferredNaming<'py> { /// Build physical locations for all models and seeds under a deferred target. #[pyfunction] #[pyo3(signature = (**options))] -fn build_deferred_locations<'py>( +pub(crate) fn build_deferred_locations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -186,7 +186,7 @@ fn planner_error(py: Python<'_>, message: String) -> PyErr { /// Validate and resolve the deferred target config. #[pyfunction] #[pyo3(signature = (**options))] -fn resolve_deferred_target_config<'py>( +pub(crate) fn resolve_deferred_target_config<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -233,7 +233,7 @@ fn deferred_target_config<'py>( /// Gather existing relations from the deferred target's relation-location schemas. #[pyfunction] #[pyo3(signature = (**options))] -fn gather_deferred_relations<'py>( +pub(crate) fn gather_deferred_relations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index d422e28f3c..256315d1ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -165,7 +165,7 @@ struct DiscoveryRequest { /// Load all raw project inputs from disk before semantic resolution. #[pyfunction] #[pyo3(signature = (**options))] -fn discover_project_inputs<'py>( +pub(crate) fn discover_project_inputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 9ffdda1e23..64231949f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -84,7 +84,7 @@ pub(crate) fn compile_analysis_selection<'py>( /// Build a static dependency graph for an already-compiled project. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph_from_compiled_project<'py>( +pub(crate) fn build_project_graph_from_compiled_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -136,7 +136,7 @@ fn plan_has_executable_work<'py>( /// Wrap a focused command projection in the established pipeline envelope. #[pyfunction] #[pyo3(signature = (**options))] -fn build_static_pipeline_result<'py>( +pub(crate) fn build_static_pipeline_result<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 142b6a0df1..89409309b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -1,5 +1,12 @@ //! Plan a compiled project over an open warehouse connection, and open that connection. +use crate::bindings::_helpers::compile_driver::deferral::build_deferred_locations; +use crate::bindings::_helpers::compile_driver::deferral::gather_deferred_relations; +use crate::bindings::_helpers::compile_driver::deferral::resolve_deferred_target_config; +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::python_plan_entries::python_read_source_names_for; +use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; @@ -50,20 +57,23 @@ pub(crate) fn pipeline_result<'py>( .getattr("resolve_python_run_selectors")? .is_truthy()? { - let graph: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildProjectGraphFromCompiledProject)?, - &[("project", project.clone())], + let graph: Bound<'py, PyAny> = build_project_graph_from_compiled_project( + py, + Some(keywords(py, &[("project", project.clone())])?), )?; run_selection = run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; selected_sql_keys = run_selection.getattr("sql_keys")?; selected_python_node_names = run_selection.getattr("python_node_names")?; - python_read_source_names = call_with( - ®istry::native(py, NativeAttr::PythonReadSourceNamesFor)?, - &[ - ("discovered_inputs", discovered.clone()), - ("python_node_names", selected_python_node_names.clone()), - ], + python_read_source_names = python_read_source_names_for( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("python_node_names", selected_python_node_names.clone()), + ], + )?), )?; } let selection: Bound<'py, PyAny> = call_with( @@ -179,41 +189,56 @@ fn deferral<'py>( return Ok((py.None().into_bound(py), py.None().into_bound(py))); } let adapter: &Bound<'py, PyAny> = inputs.adapter; - let target_config: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveDeferredTargetConfig)?, - &[ - ("discovered_inputs", inputs.discovered_inputs.clone()), - ("defer_to", defer_to.clone()), - ( - "current_target_name", - project.getattr("effective_target_name")?, - ), - ], + let target_config: Bound<'py, PyAny> = resolve_deferred_target_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", inputs.discovered_inputs.clone()), + ("defer_to", defer_to.clone()), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?), )?; - let locations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildDeferredLocations)?, - &[ - ("project", project.clone()), - ("deferred_target_config", target_config), - ("effective_vars", project.getattr("effective_vars")?), - ("default_schema", adapter.call_method0("default_schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ], + let locations: Bound<'py, PyAny> = object( + py, + build_deferred_locations( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("deferred_target_config", target_config), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + )?), + )?, )?; - let relations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::GatherDeferredRelations)?, - &[ - ("adapter", adapter.clone()), - ("connection", inputs.connection.clone()), - ("deferred_locations", locations.clone()), - ], + let relations: Bound<'py, PyAny> = object( + py, + gather_deferred_relations( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("connection", inputs.connection.clone()), + ("deferred_locations", locations.clone()), + ], + )?), + )?, )?; Ok((locations, relations)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 276d018fa5..ca9031ae7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -71,7 +71,7 @@ fn py_bool(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { /// Build one compiled project with adapter defaults and target validation applied. #[pyfunction] #[pyo3(signature = (**options))] -fn build_compiled_project<'py>( +pub(crate) fn build_compiled_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -318,7 +318,7 @@ fn build_project_graph_with_analysis_selection<'py>( /// Build the static dependency graph for a compiled project. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph<'py>( +pub(crate) fn build_project_graph<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 387db3dbef..625a1a4cac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -1,5 +1,6 @@ //! Display-ready Python plan entries and Python-aware selection warnings of a plan. +use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -15,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const SKIPPED_UPSTREAM_CODE: &str = "P501"; @@ -68,7 +69,7 @@ fn node_graph<'py>(discovered: &Bound<'py, PyAny>) -> PyResult /// Return sources declared by the selected Python nodes and their Python upstreams. #[pyfunction] #[pyo3(signature = (**options))] -fn python_read_source_names_for<'py>( +pub(crate) fn python_read_source_names_for<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -104,7 +105,7 @@ fn python_read_source_names_for<'py>( /// Attach python-aware selection warnings and plan entries to the plan output. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_run_plan_outputs<'py>( +pub(crate) fn build_python_run_plan_outputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -161,10 +162,7 @@ fn build_python_run_plan_outputs<'py>( ("python_node_names", selected.clone()), ], )?; - let lifecycle: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonSqlRunLifecycle)?, - &[("selection", selection), ("python_graph", graph.clone())], - )?; + let lifecycle: Bound<'py, PyAny> = build_python_sql_run_lifecycle(&selection, &graph)?; let entries: Bound<'py, PyTuple> = plan_entries( &lifecycle, (&graph, &kinds), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index ff9880d2d4..03045bde39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -47,7 +47,7 @@ fn shared_error(py: Python<'_>, message: String) -> PyErr { /// Return adapter-qualified runtime relations required by selected Python nodes. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_relation_targets<'py>( +pub(crate) fn build_python_relation_targets<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -80,7 +80,7 @@ fn build_python_relation_targets<'py>( /// Return every planned project relation for hard-coded name warnings, or None when off. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_relation_targets<'py>( +pub(crate) fn build_project_relation_targets<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 65b0c4e43c..7776309998 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -211,7 +211,7 @@ fn deferred_locations<'py>( /// Run canonical compile, selector, and relation phases without build planning. #[pyfunction] #[pyo3(signature = (**request))] -fn compile_static_command_context<'py>( +pub(crate) fn compile_static_command_context<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request: Request<'_, 'py> = Request::new("compile_static_command_context", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index dd1c27fc9f..731a22f154 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -309,7 +309,7 @@ fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult( +pub(crate) fn matching_dynamic_families<'py>( families: &Bound<'py, PyAny>, column_name: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 2a940a21e7..fadaa324d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -63,7 +63,7 @@ pub(crate) fn interval_validation( /// The canonical partition containing an observed physical value. #[pyfunction(name = "cursor_observed_partition")] #[pyo3(signature = (*, value, grain))] -fn partition<'py>( +pub(crate) fn partition<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index 3648587691..8225c1b08b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -187,7 +187,7 @@ pub(crate) fn as_scalar<'py>( /// One cursor scalar, preserving date-ness and timestamp offsets. #[pyfunction(name = "cursor_parse")] #[pyo3(signature = (*, raw, cursor_type))] -fn parse<'py>( +pub(crate) fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -248,7 +248,7 @@ fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult, right: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { let session: CursorSession<'_> = CursorSession::new(left.py())?; let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); Ok(match session.checked(compare(&left, &right, &session))? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 4fadf5da43..0cd7da5cd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -48,7 +48,7 @@ fn sql_analysis_off_guidance( } #[pyfunction] -fn microbatch_guidance() -> (String, String) { +pub(crate) fn microbatch_guidance() -> (String, String) { concurrency_guidance() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index 976d68d0ef..f0fd7178d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -8,7 +8,7 @@ use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; /// The 1-based line of the outermost `- name: ` entry, if present. #[pyfunction] #[pyo3(signature = (*, contents, name))] -fn yaml_entry_line(contents: &str, name: &str) -> Option { +pub(crate) fn yaml_entry_line(contents: &str, name: &str) -> Option { yaml_entry_lines(contents) .into_iter() .find_map(|(entry, line)| (entry == name).then_some(line)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index cc20c8e1c3..398a2e3074 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -56,7 +56,7 @@ pub(crate) fn fingerprint_set_from_rows<'py>( /// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. #[pyfunction] #[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] -fn select_latest_fingerprints<'py>( +pub(crate) fn select_latest_fingerprints<'py>( py: Python<'py>, fingerprint_set: &Bound<'py, PyAny>, node_names: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs index dbf133c673..846cc60777 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod latest_rows; -mod query_hashes; +pub(crate) mod query_hashes; pub(crate) mod registration; -mod storage_io; +pub(crate) mod storage_io; pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs index 9532cbf23b..743cc60100 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs @@ -27,7 +27,11 @@ fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { /// The persisted SHA-256 fingerprint of the query's canonical token stream. #[pyfunction] #[pyo3(signature = (*, query_sql, dialect))] -fn compute_query_hash(py: Python<'_>, query_sql: &str, dialect: Option<&str>) -> PyResult { +pub(crate) fn compute_query_hash( + py: Python<'_>, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) .map_err(|error| query_fingerprint_error(py, error)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 01ea896109..10890995c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -291,7 +291,7 @@ fn unreadable_table(py: Python<'_>, qualified_name: &str, error: PyErr) -> PyErr /// The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. #[pyfunction] #[pyo3(signature = (**request))] -fn read_latest_fingerprints<'py>( +pub(crate) fn read_latest_fingerprints<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("read_latest_fingerprints", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 1d6b3465e9..854795f961 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -261,7 +261,7 @@ fn execute_sql( /// Append source freshness rows, creating the table once if needed. #[pyfunction] #[pyo3(signature = (**options))] -fn write_source_freshness_records<'py>( +pub(crate) fn write_source_freshness_records<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index 28dd9b0966..2b1fda2282 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -50,7 +50,7 @@ fn neighbours<'py>( /// Keys within `max_depth` edges of any start, plus the starts when requested. #[pyfunction] #[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] -fn transitive_closure_many<'py>( +pub(crate) fn transitive_closure_many<'py>( starts: &Bound<'py, PyAny>, edges: &Bound<'py, PyAny>, include_starts: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index d81e6923b7..603a7870aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -118,7 +118,10 @@ fn model_python_hook_names<'py>( /// The compiled graph key a typed model, source or seed reference names. #[pyfunction] -fn sql_ref_key<'py>(py: Python<'py>, reference: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn sql_ref_key<'py>( + py: Python<'py>, + reference: &Bound<'py, PyAny>, +) -> PyResult> { let resource_type: Bound<'py, PyAny> = py .import(COMPILE_TYPES_MODULE)? .getattr("CompiledResourceType")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 40cb605076..e0d28601d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,9 +1,9 @@ pub(crate) mod algorithms; -mod execution_gates; +pub(crate) mod execution_gates; pub(crate) mod keys; pub(crate) mod node_set; pub(crate) mod project_graph; -mod python_lifecycle; +pub(crate) mod python_lifecycle; pub(crate) mod python_node_graph; pub(crate) mod python_selection; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs index dc4c16371c..d08a2c659b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -15,7 +15,7 @@ const EXTERNAL_CONNECTION_MODE: &str = "external"; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. #[pyfunction] #[pyo3(signature = (*, selection, python_graph))] -fn build_python_sql_run_lifecycle<'py>( +pub(crate) fn build_python_sql_run_lifecycle<'py>( selection: &Bound<'py, PyAny>, python_graph: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index b8b1d112d7..851997d86b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -143,7 +143,7 @@ fn resolve_python_node_selection<'py>( /// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. #[pyfunction] #[pyo3(signature = (*, graph, exclude))] -fn resolve_python_check_excludes<'py>( +pub(crate) fn resolve_python_check_excludes<'py>( graph: &Bound<'py, PyAny>, exclude: Vec, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 1838d1a276..9aaeeeac51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -1,5 +1,7 @@ //! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -87,17 +89,20 @@ pub(crate) fn compile_janitor_project<'py>( let project: Bound<'py, PyAny> = within(&lifecycle(py, ("project", "project_compile"))?, |_| { status.call_method1("on_progress", ("Compiling project...",))?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ( - "discovered_inputs", - invocation.getattr("discovered_inputs")?, - ), - ("adapter", context.getattr("adapter")?), - ("selected_target", target), - ("resolved_connection", context.getattr("connection_config")?), - ], + let project: Bound<'py, PyAny> = build_compiled_project( + py, + Some(keywords( + py, + &[ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("adapter", context.getattr("adapter")?), + ("selected_target", target), + ("resolved_connection", context.getattr("connection_config")?), + ], + )?), )?; status.call_method1( "on_progress", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 0d964e8bab..389dbc19b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -91,7 +91,7 @@ pub(crate) fn inclusive_cursor_end( /// A fully bounded inclusive operator override as an executable interval. #[pyfunction] #[pyo3(signature = (*, start_cursor_override, end_cursor_override, cursor_type, cursor_grain))] -fn resolve_bounded_cursor_override<'py>( +pub(crate) fn resolve_bounded_cursor_override<'py>( start_cursor_override: &Bound<'py, PyAny>, end_cursor_override: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 09b0ad43d3..ac8e45266c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,6 +1,7 @@ //! Detect how each selected model and function changed since its recorded build. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -42,7 +43,7 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; @@ -156,7 +157,7 @@ pub(crate) fn detect_changes<'py>( /// Selected model changes, without inspecting functions. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_model_changes_in_scope<'py>( +pub(crate) fn detect_model_changes_in_scope<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -190,7 +191,7 @@ fn detect_model_changes_in_scope<'py>( /// One model's change and backfill. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_model_changes<'py>( +pub(crate) fn detect_model_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -594,23 +595,15 @@ fn schema_findings_of<'py>( (type_enforcement, inferred_schema_complete): (bool, bool), (dynamic, dialect): (Option>, &Bound<'py, PyAny>), ) -> PyResult> { - let py: Python<'py> = dialect.py(); + let _py: Python<'py> = dialect.py(); let dialect_name: Option = dialect.extract()?; let mut types_equal = |left: &str, right: &str| -> PyResult { native_types_equal(dialect, left, right) }; - let matching_function: Bound<'py, PyAny> = - registry::native(py, NativeAttr::MatchingDynamicFamilies)?; let mut single_family_type = |name: &str| -> PyResult> { let Some(families) = &dynamic else { return Ok(None); }; - let matching: Bound<'py, PyAny> = call_with( - &matching_function, - &[ - ("families", families.clone()), - ("column_name", PyString::new(py, name).into_any()), - ], - )?; + let matching = matching_dynamic_families(families, name)?; if matching.len()? != 1 { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs index e760bb22da..2320bd9193 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs @@ -35,7 +35,7 @@ fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) /// Fingerprint metadata JSON without the declared-columns hash key, if present. #[pyfunction] -fn without_declared_columns_hash(metadata_json: &str) -> String { +pub(crate) fn without_declared_columns_hash(metadata_json: &str) -> String { without_hash(metadata_json) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 47462cf513..12d79a6396 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -1,9 +1,10 @@ //! Model renames that downstream `__ref` names may have been rewritten for. +use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; @@ -16,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const GENERIC_DIALECT: &str = "generic"; @@ -47,16 +48,9 @@ pub(crate) fn origin_names( .filter(|name| !name.is_empty()) .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); let mut spans = |names: &BTreeSet| ref_name_spans(&query_sql, &dialect_name, names); - let compute_hash: Bound<'_, PyAny> = registry::native(py, NativeAttr::ComputeQueryHash)?; + let dialect_option: Option = dialect.extract()?; let mut query_hash = |text: &str| -> PyResult { - call_with( - &compute_hash, - &[ - ("query_sql", PyString::new(py, text).into_any()), - ("dialect", dialect.clone()), - ], - )? - .extract() + compute_query_hash(py, text, dialect_option.as_deref()) }; origin_reference_names( (&query_sql, recorded_hash), @@ -143,7 +137,7 @@ fn origin_reference_names_of<'py>( /// This plan's renames plus recorded renames, read only when a selected query that references models changed. #[pyfunction] #[pyo3(signature = (**options))] -fn plan_reference_renames<'py>( +pub(crate) fn plan_reference_renames<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index d5d677a858..3015e9551c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -2,14 +2,30 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_upstream_deps, +}; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use pyo3::PyRef; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -50,17 +66,17 @@ fn clone_plan_output<'py>( exclude: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - let downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; - let selected = native( + let upstream = build_execution_upstream_deps(project)?.into_any(); + let downstream = build_downstream_deps(&upstream)?.into_any(); + let selected = object( py, - NativeAttr::ResolveGraphSelection, - &[ - ("graph", project.getattr("lineage_graph")?), - ("select", select.clone()), - ("exclude", exclude.clone()), - ], + resolve_graph_selection( + project + .getattr("lineage_graph")? + .extract::>()?, + select, + exclude, + )?, )?; planner_model( py, @@ -68,11 +84,7 @@ fn clone_plan_output<'py>( &[ ( "execution_order", - native( - py, - NativeAttr::TopologicallyOrderKeys, - &[("upstream", upstream.clone())], - )?, + object(py, topologically_order_keys(&upstream, None)?)?, ), ("selected_keys", selected), ("upstream_deps", upstream), @@ -96,34 +108,20 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = native( - py, - NativeAttr::BuildModelLocations, - &[("models", project.getattr("models")?)], - )?; - let seed_locations = native( - py, - NativeAttr::BuildSeedLocations, - &[("seeds", project.getattr("seeds")?)], - )?; - let function_locations = native( + let model_locations = object(py, build_model_locations(&project.getattr("models")?)?)?; + let seed_locations = object(py, build_seed_locations(&project.getattr("seeds")?)?)?; + let function_locations = object( py, - NativeAttr::BuildFunctionLocations, - &[("functions", project.getattr("functions")?)], + build_function_locations(&project.getattr("functions")?)?, )?; let source_map = PyDict::new(py); for source in source_project.getattr("sources")?.try_iter()? { let source = source?; source_map.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; } - let source_warehouse_columns = native( + let source_warehouse_columns = object( py, - NativeAttr::GatherSourceColumns, - &[ - ("project", source_project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ], + gather_source_columns(source_project, adapter, connection, None)?, )?; Ok(CloneResolution { model_locations, @@ -180,11 +178,7 @@ fn clone_model_entries<'py>( if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; } - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; + let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); for function in project.getattr("functions")?.try_iter()? { @@ -312,25 +306,27 @@ fn clone_function_entries<'py>( if !selected.contains(function.getattr("key")?)? { continue; } - let body_sql = native( + let body_sql = resolve_function_sql( py, - NativeAttr::ResolveFunctionSql, - &[ - ("adapter", adapter.clone()), - ("function", function.clone()), - ("model_locations", resolution.model_locations.clone()), - ("seed_locations", resolution.seed_locations.clone()), - ("function_locations", resolution.function_locations.clone()), - ("source_map", resolution.source_map.clone()), - ( - "source_warehouse_columns", - resolution.source_warehouse_columns.clone(), - ), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ], + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", resolution.model_locations.clone()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?), )?; let field = |name: &str| function.getattr(name); let entry = planner_model( @@ -346,11 +342,7 @@ fn clone_function_entries<'py>( ("body_sql", body_sql), ( "fingerprint_query_sql", - native( - py, - NativeAttr::BuildCompiledFunctionFingerprintSql, - &[("function", function.clone())], - )?, + object(py, build_compiled_function_fingerprint_sql(&function)?)?, ), ("fingerprint_destination", field("fingerprint_destination")?), ("return_columns", field("return_columns")?), @@ -415,11 +407,7 @@ fn origin_model_entries<'py>( if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; } - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; + let materialization_type = materialization_of(&model)?; entries.push(model_entry(&model, materialization_type, string(py, ""))?); } tuple_of(py, entries) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 1930333df0..041d7a9afd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -2,15 +2,24 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_entries::values::{ - boolean, native, none, planner_model, planner_type, + boolean, none, planner_model, planner_type, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -82,21 +91,14 @@ fn resolve_static_command_scope<'py>( auto_load_sources: bool, ) -> PyResult> { let py = project.py(); - native( - py, - NativeAttr::ResolvePlannerScopes, - &[ - ("project", project.clone()), - ("selection", selection.clone()), - ( - "policies", - planner_model( - py, - RecordId::PlannerPolicies, - &[("auto_load_sources", boolean(py, auto_load_sources)?)], - )?, - ), - ], + resolve_planner_scopes( + project, + selection, + &planner_model( + py, + RecordId::PlannerPolicies, + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, )? .getattr("selected_scope") } @@ -150,24 +152,21 @@ fn build_audit_command_plan<'py>( let model = model?; materializations.set_item( model_reads::name_object(&model)?, - native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )? - .getattr("value")?, + materialization_of(&model)?.getattr("value")?, )?; } - let audits = native( + let audits = object( py, - NativeAttr::BuildSelectedAuditEntries, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, )?; base_plan_output( project, @@ -235,11 +234,7 @@ fn build_test_command_plan<'py>( if !function_keys.contains(function.getattr("key")?)? { continue; } - let fingerprint_sql = native( - py, - NativeAttr::BuildCompiledFunctionFingerprintSql, - &[("function", function.clone())], - )?; + let fingerprint_sql = object(py, build_compiled_function_fingerprint_sql(&function)?)?; functions.push(function_entry( &function, &adapter, @@ -299,7 +294,7 @@ fn build_seed_command_plan<'py>( if !selected_keys.contains(seed.getattr("key")?)? { continue; } - let identity = native(py, NativeAttr::BuildSeedIdentity, &[("seed", seed.clone())])?; + let identity = object(py, build_seed_identity(&seed)?)?; let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); let previous = existing.call_method1("get", (seed.getattr("name")?,))?; let reason = if previous.is_none() { @@ -320,14 +315,7 @@ fn build_seed_command_plan<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - native( - py, - NativeAttr::ExtractSeedColumns, - &[("seed", seed.clone())], - )?, - ), + ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -353,7 +341,7 @@ fn build_seed_command_plan<'py>( /// Project static relation data for Python checks. #[pyfunction] #[pyo3(signature = (*, project, scope, relations))] -fn build_relation_command_plan<'py>( +pub(crate) fn build_relation_command_plan<'py>( project: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -415,9 +403,8 @@ fn read_selected_seed_fingerprints<'py>( } } let node_names = PyTuple::new(py, names.iter())?; - let latest = native( + let latest = read_latest_fingerprints(Some(&keywords( py, - NativeAttr::ReadLatestFingerprints, &[ ("connection", connection.clone()), ("execute", adapter.getattr("execute")?), @@ -439,7 +426,7 @@ fn read_selected_seed_fingerprints<'py>( ), ("node_names", node_names.into_any()), ], - )?; + )?))?; let items = PyList::new( py, latest diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index b56f6ed178..48509ee8c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -5,8 +5,13 @@ use crate::bindings::_helpers::model_config::config_values::config_str as author use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_upstream_deps, +}; +use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -46,18 +51,10 @@ fn display_entry<'py>( full_refresh: bool, ) -> PyResult> { let py = model.py(); - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; - let effective = native( + let materialization_type = materialization_of(model)?; + let effective = object( py, - NativeAttr::ResolveModelFullRefresh, - &[ - ("model", model.clone()), - ("cli_full_refresh", boolean(py, full_refresh)?), - ], + resolve_model_full_refresh(model, boolean(py, full_refresh)?.is_truthy()?)?, )? .is_truthy()?; let query_sql = model_reads::query_sql(model)?; @@ -151,9 +148,8 @@ fn build_display_only_sqlbuild_plan<'py>( .iter() .map(|entry| entry.getattr("key")) .collect::>()?; - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - let downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; + let upstream = build_execution_upstream_deps(project)?.into_any(); + let downstream = build_downstream_deps(&upstream)?.into_any(); planner_model( py, RecordId::PlanOutput, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 068538d9ca..4cf714f6d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -183,7 +183,7 @@ fn planned_output<'py>( /// Build the execution plan for one selection against live warehouse state. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_execution_plan<'py>( +pub(crate) fn build_execution_plan<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 5e31964014..a8dddfd37f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,10 +1,10 @@ //! Plan entry and plan output assembly over native planner resolution. mod clone_plans; -mod command_plans; +pub(crate) mod command_plans; pub(crate) mod contexts; mod display_plan; -mod execution_plan; +pub(crate) mod execution_plan; pub(crate) mod model_entries; pub(crate) mod models; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 1355be400e..63889b31af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,8 +1,11 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; +use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_entries::models::{ ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, }; @@ -15,6 +18,7 @@ use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -24,6 +28,7 @@ use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as Sq use crate::bindings::_helpers::planner_resolution::orchestration::{ Resolution, model_hook_entries, resolve_model_sql, }; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names; use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ build_cursor_input_relations, reject_unexposed_cursor_input_columns, }; @@ -37,8 +42,9 @@ use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; use crate::bindings::_helpers::planner_strategy::values::text; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::semantic_shapes::source_bindings::source_binding_diagnostics; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -228,23 +234,25 @@ fn check_binding_diagnostics<'py>( errors.push(diagnostic); } } - let bound = native( + let bound = object( py, - NativeAttr::SourceBindingDiagnostics, - &[ - ("project", project.clone()), - ( - "columns", - class_field(relations, |context: &PlannerRelationsContext| { - &context.source_warehouse_columns - })?, - ), - ( - "profile", - adapter.call_method0("expression_inference_profile")?, - ), - ("selected_keys", selected_keys.clone()), - ], + source_binding_diagnostics(Some(&keywords( + py, + &[ + ("project", project.clone()), + ( + "columns", + class_field(relations, |context: &PlannerRelationsContext| { + &context.source_warehouse_columns + })?, + ), + ( + "profile", + adapter.call_method0("expression_inference_profile")?, + ), + ("selected_keys", selected_keys.clone()), + ], + )?))?, )?; for diagnostic in bound.try_iter()? { let diagnostic = diagnostic?; @@ -307,7 +315,7 @@ fn check_microbatch_concurrency<'py>( } let settings = project.getattr("settings")?; if !configured.is_empty() && !settings.getattr("microbatch_concurrency")?.is_truthy()? { - let guidance = native(py, NativeAttr::MicrobatchGuidance, &[])?; + let guidance = object(py, microbatch_guidance())?; let prefix: &str = MICROBATCH_CONCURRENCY_MESSAGE; return Err(error_of( py, @@ -408,13 +416,9 @@ pub(crate) fn plan_entries<'py>( let external_blocked = build.external_blocked_model_names.bind(py).clone(); let start_override = build.start_cursor_override.bind(py).clone(); let end_override = build.end_cursor_override.bind(py).clone(); - let producers = native( + let producers = object( py, - NativeAttr::RuntimeCursorProducerNames, - &[ - ("scope", scope.clone()), - ("source_map", relations_context.source_map.bind(py).clone()), - ], + runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); for function in project.getattr("functions")?.try_iter()? { @@ -534,7 +538,7 @@ pub(crate) fn plan_entries<'py>( /// Detect changes and build a plan entry and warnings for a single model. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn plan_model<'py>( +pub(crate) fn plan_model<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -558,19 +562,21 @@ fn plan_model<'py>( )?; let model = args.value("model")?; let snapshot = args.value("snapshot")?; - let change_result = native( + let change_result = detect_model_changes( py, - NativeAttr::DetectModelChanges, - &[ - ("model", model.clone()), - ("snapshot", snapshot.clone()), - ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), - ( - "query_change_tracking", - args.value("query_change_tracking")?, - ), - ("full_refresh", args.value("full_refresh")?), - ], + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?), )?; let overrides = args.value("cursor_overrides")?; plan_model_from_change(&ModelPlanRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index d4f97a8f78..e4b9d2fd82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,13 +1,20 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; +use crate::bindings::_helpers::planner_scope::loaders::upstream_loader_dependency_names; +use crate::bindings::_helpers::planner_scope::sources::build_source_load_entries; +use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; @@ -119,14 +126,7 @@ fn seed_entries<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - native( - py, - NativeAttr::ExtractSeedColumns, - &[("seed", seed.clone())], - )?, - ), + ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -290,13 +290,9 @@ fn require_intermediate_loader_targets<'py>( else { continue; }; - let dependencies = native( + let dependencies = object( py, - NativeAttr::UpstreamLoaderDependencyNames, - &[ - ("loader_function", loader_function), - ("loader_functions", loader_functions.clone()), - ], + upstream_loader_dependency_names(&loader_function, &loader_functions)?, )?; for dependency_name in dependencies.try_iter()? { let dependency_name = dependency_name?; @@ -343,28 +339,30 @@ pub(super) fn function_entry<'py>( ), ) -> PyResult> { let py = function.py(); - let body_sql = native( + let body_sql = resolve_function_sql( py, - NativeAttr::ResolveFunctionSql, - &[ - ("adapter", adapter.clone()), - ("function", function.clone()), - ("model_locations", relations.getattr("model_locations")?), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", relations.getattr("source_read_map")?), - ( - "source_warehouse_columns", - relations.getattr("source_warehouse_columns")?, - ), - ( - "star_exclude_keyword", - relations.getattr("star_exclude_keyword")?, - ), - ], + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ], + )?), )?; let field = |name: &str| function.getattr(name); planner_model( @@ -565,15 +563,14 @@ pub(crate) fn plan_output_fields<'py>( let execution_order = scope.getattr("execution_order")?; let source_map = relations.getattr("source_map")?; let seeds = seed_entries(&project, &snapshot, &selected_keys, &extras)?; - let source_load_entries = native( + let source_load_entries = object( py, - NativeAttr::BuildSourceLoadEntries, - &[ - ("execution_order", execution_order.clone()), - ("selected_keys", selected_keys.clone()), - ("source_map", source_map.clone()), - ("is_reload", reload_sources), - ], + build_source_load_entries( + &execution_order, + &selected_keys, + source_map.downcast::()?, + reload_sources.is_truthy()?, + )?, )?; require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; let functions = function_entries( @@ -589,16 +586,18 @@ pub(crate) fn plan_output_fields<'py>( entry.getattr("materialization_type")?, )?; } - let audit_entries = native( + let audit_entries = object( py, - NativeAttr::BuildSelectedAuditEntries, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, )?; let selection = scope.getattr("sql_test_selection")?; let mut planned = extras.getattr("planned_sql_tests")?; @@ -717,7 +716,7 @@ fn plan_function<'py>( /// Plan SQL tests selected by name or overlapping one selection into a reusable result. #[pyfunction] #[pyo3(signature = (*, project, adapter, selected_keys, sql_test_selection))] -fn plan_selected_sql_tests<'py>( +pub(crate) fn plan_selected_sql_tests<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 94c0e1bf6c..d67d2d00b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,12 +1,23 @@ //! Relation locations and source metadata for plan entry construction. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::models::{ PlannerRelationsContext, RelationsRequest, }; -use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; +use crate::bindings::_helpers::planner_entries::values::{none, planner_model}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; +use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; +use crate::bindings::_helpers::planner_scope::sources::build_source_load_map; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; +use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; +use crate::bindings::_helpers::planner_warehouse::source_deferral::build_source_read_map; +use crate::bindings::_helpers::planner_warehouse::source_deferral::with_declared_source_reads; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -54,15 +65,11 @@ fn source_warehouse_columns<'py>( .try_iter()? .collect::>>()?, )?; - let gathered = native( - py, - NativeAttr::GatherSourceInspection, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("source_entries", entries.into_any()), - ], + let gathered = gather_source_inspection( + project, + adapter, + connection, + entries.into_any().extract::>>()?, )?; Ok((gathered.get_item(0)?, gathered.get_item(1)?)) } @@ -89,32 +96,22 @@ pub(crate) fn planner_relations_context<'py>( deferral }; let selected_keys = scope.getattr("selected_keys")?; - let mut model_locations = native( + let mut model_locations = object(py, build_model_locations(&project.getattr("models")?)?)?; + let mut seed_locations = object(py, build_seed_locations(&project.getattr("seeds")?)?)?; + let function_locations = object( py, - NativeAttr::BuildModelLocations, - &[("models", project.getattr("models")?)], - )?; - let mut seed_locations = native( - py, - NativeAttr::BuildSeedLocations, - &[("seeds", project.getattr("seeds")?)], - )?; - let function_locations = native( - py, - NativeAttr::BuildFunctionLocations, - &[("functions", project.getattr("functions")?)], + build_function_locations(&project.getattr("functions")?)?, )?; let deferred_locations = deferral.getattr("deferred_locations")?; if !deferred_locations.is_none() { - let applied = native( + let applied = object( py, - NativeAttr::ApplyDeferredLocations, - &[ - ("model_locations", model_locations), - ("seed_locations", seed_locations), - ("deferred_locations", deferred_locations), - ("selected_keys", selected_keys.clone()), - ], + apply_deferred_locations( + model_locations.downcast::()?, + seed_locations.downcast::()?, + deferred_locations.downcast::()?, + &selected_keys, + )?, )?; model_locations = applied.get_item(0)?; seed_locations = applied.get_item(1)?; @@ -125,24 +122,23 @@ pub(crate) fn planner_relations_context<'py>( } else { relation_keys }; - let source_map = native( + let source_map = object( py, - NativeAttr::BuildSourceLoadMap, - &[ - ("project", project.clone()), - ("selected_keys", load_keys), - ( - "upstream_deps", - if scope.getattr("all_keys")?.is_truthy()? { - scope.getattr("upstream_deps")? - } else { - none(py) - }, - ), - ], + build_source_load_map( + &project, + &load_keys, + Some(&if scope.getattr("all_keys")?.is_truthy()? { + scope.getattr("upstream_deps")? + } else { + none(py) + }) + .filter(|value| !value.is_none()), + )?, )?; let deferral_enabled = deferral.getattr("source_deferral_enabled")?.is_truthy()?; - let read_map = |map: &Bound<'py, PyAny>, names: Option>| { + let read_map = |map: &Bound<'py, PyAny>, + names: Option>| + -> PyResult> { if !deferral_enabled { return Ok(map.clone()); } @@ -157,17 +153,16 @@ pub(crate) fn planner_relations_context<'py>( if let Some(names) = names { pairs.push(("read_source_names", names)); } - native(py, NativeAttr::BuildSourceReadMap, &pairs) + Ok(build_source_read_map(py, Some(keywords(py, &pairs)?))?.into_any()) }; let source_read_map = read_map(&source_map, None)?; - let python_source_map = native( + let python_source_map = object( py, - NativeAttr::WithDeclaredSourceReads, - &[ - ("project", project.clone()), - ("source_map", source_map.clone()), - ("source_names", scope.getattr("python_read_source_names")?), - ], + with_declared_source_reads( + &project, + source_map.downcast::()?, + &scope.getattr("python_read_source_names")?, + )?, )?; let python_names = PyFrozenSet::new( py, @@ -182,18 +177,21 @@ pub(crate) fn planner_relations_context<'py>( &source_read_map, &known_source_columns, )?; - let semantic_columns = native( + let semantic_columns = semantic_source_columns( py, - NativeAttr::GetSemanticSourceColumns, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection), - ("source_read_map", source_read_map.clone()), - ("selected_keys", selected_keys), - ("columns", columns), - ], - )?; + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection), + ("source_read_map", source_read_map.clone()), + ("selected_keys", selected_keys), + ("columns", columns), + ], + )?), + )? + .into_any(); Ok(Bound::new( py, PlannerRelationsContext { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 2b47455ce3..6335cc5e60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -80,7 +80,9 @@ fn hook_label(hook_name: &str, kind: &str) -> String { /// Execution-order edges from lineage and woven SQL test nodes. #[pyfunction] -fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_execution_upstream_deps<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project.py(); let upstream: Bound<'py, PyDict> = PyDict::new(py); for collection in ["models", "sources", "seeds", "functions"] { @@ -162,7 +164,9 @@ fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_execution_edge_origins<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project.py(); let origins: Bound<'py, PyDict> = PyDict::new(py); for test in project.getattr("sql_tests")?.try_iter()? { @@ -219,7 +223,9 @@ fn build_execution_edge_origins<'py>(project: &Bound<'py, PyAny>) -> PyResult(upstream: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_downstream_deps<'py>( + upstream: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = upstream.py(); let result: Bound<'py, PyDict> = PyDict::new(py); for (key, sources) in inverted(upstream)?.iter() { @@ -265,7 +271,7 @@ fn edge_origins<'py>( /// All keys in stable dependency order using Kahn's algorithm. #[pyfunction] #[pyo3(signature = (*, upstream, injected_edge_origins = None))] -fn topologically_order_keys<'py>( +pub(crate) fn topologically_order_keys<'py>( upstream: &Bound<'py, PyAny>, injected_edge_origins: Option>, ) -> PyResult> { @@ -327,7 +333,7 @@ fn missing_rows<'py>( /// Upstream deps of selected keys that exist neither in scope nor in the warehouse. #[pyfunction] #[pyo3(signature = (**options))] -fn check_buildability<'py>( +pub(crate) fn check_buildability<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -386,7 +392,7 @@ fn check_buildability<'py>( /// Describe missing upstream keys, naming the audit when only an audit read requires one. #[pyfunction] #[pyo3(signature = (*, missing, edge_origins))] -fn missing_upstream_message<'py>( +pub(crate) fn missing_upstream_message<'py>( missing: &Bound<'py, PyAny>, edge_origins: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs index ca508ec8b1..07a5cc44d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -75,7 +75,7 @@ pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { /// Fingerprint a model query for change detection, naming the model when it cannot. #[pyfunction] #[pyo3(signature = (*, model_name, query_sql, dialect))] -fn model_definition_hash( +pub(crate) fn model_definition_hash( py: Python<'_>, model_name: &str, query_sql: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 9ecd1ba24a..5f93834e0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -64,7 +64,7 @@ fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn build_direct_model_version_identities<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -201,7 +201,7 @@ fn build_direct_model_version_identities<'py>( /// Planned write hashes replaced by hashes based on upstreams available this run. #[pyfunction] #[pyo3(signature = (**options))] -fn with_honest_model_write_hashes<'py>( +pub(crate) fn with_honest_model_write_hashes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -354,7 +354,7 @@ fn with_honest_model_write_hashes<'py>( /// Resolved actions with recomputed model changes patched in where available. #[pyfunction] #[pyo3(signature = (*, resolved_actions, changes))] -fn merge_recomputed_model_changes<'py>( +pub(crate) fn merge_recomputed_model_changes<'py>( resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 35d06a6406..a909523176 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -456,7 +456,7 @@ fn build_version_identity_metadata_json<'py>( /// Deterministic non-query model fingerprint metadata JSON. #[pyfunction] #[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] -fn build_model_version_identity_metadata_json<'py>( +pub(crate) fn build_model_version_identity_metadata_json<'py>( model: &Bound<'py, PyAny>, function_local_hashes: Option>, hook_version_hashes: Option>, @@ -478,7 +478,7 @@ fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { /// Local-only semantic hashes of functions by name. #[pyfunction] #[pyo3(signature = (*, functions, dialect))] -fn build_function_local_hashes<'py>( +pub(crate) fn build_function_local_hashes<'py>( functions: &Bound<'py, PyAny>, dialect: Option<&str>, ) -> PyResult> { @@ -487,13 +487,15 @@ fn build_function_local_hashes<'py>( /// Stable function definition text used for change fingerprints. #[pyfunction] -fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn build_compiled_function_fingerprint_sql( + function: &Bound<'_, PyAny>, +) -> PyResult { compiled_function_fingerprint_sql(function) } /// `(identity_hash, metadata_json)` for one compiled seed. #[pyfunction] -fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { +pub(crate) fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { seed_identity(seed) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index 5559b2c1e6..716f47a687 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -170,7 +170,7 @@ fn plan_warnings<'py>( /// Warn for selected models stale through changed upstreams outside the run set. #[pyfunction] #[pyo3(signature = (**options))] -fn build_stale_out_of_selection_warnings<'py>( +pub(crate) fn build_stale_out_of_selection_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index 8d4f973716..4299bea43d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -2,12 +2,12 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, native, + config, definition_hash, lower, }; use crate::bindings::_helpers::planner_migrations::models::{ ColumnCandidate, ColumnMigrationInputs, MigrationPhase, }; -use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -73,24 +73,18 @@ pub(super) fn candidates<'py>( by_name.set_item(lower(&column.getattr("name")?)?, column)?; } let previous = if changed { - native( + parse_query_shape( py, - NativeAttr::ParseQueryShape, - &[ - ("query_sql", fingerprint.getattr("definition")?), - ("dialect", dialect.clone()), - ], + &fingerprint.getattr("definition")?.extract::()?, + dialect.extract::>()?.as_deref(), )? } else { py.None().into_bound(py) }; - let current = native( + let current = parse_query_shape( py, - NativeAttr::ParseQueryShape, - &[ - ("query_sql", model_reads::query_sql(&model)?), - ("dialect", dialect.clone()), - ], + &model_reads::query_sql(&model)?.extract::()?, + dialect.extract::>()?.as_deref(), )?; result.push(ColumnCandidate { model, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 668ac1047d..128f2c6273 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -3,16 +3,18 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; -use crate::bindings::_helpers::planner_migrations::column_objects::{ - lower, model, native, replace, -}; +use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; +use crate::bindings::_helpers::planner_migrations::columns::identical_renames; +use crate::bindings::_helpers::planner_migrations::columns::rename_hints; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_column; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -33,16 +35,21 @@ pub(super) fn decide<'py>( pairs.push((origin, destination, "manual")); } if !candidate.previous.is_none() && !candidate.current.is_none() { - let detected = native( + let detected = object( py, - NativeAttr::IdenticalRenames, - &[ - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("excluded", declared_names(candidate)?.into_any()), - ("declared", declared.into_any()), - ("input_columns", inputs.clone()), - ], + identical_renames( + py, + Some(keywords( + py, + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", declared_names(candidate)?.into_any()), + ("declared", declared.into_any()), + ("input_columns", inputs.clone()), + ], + )?), + )?, )?; for pair in detected.try_iter()? { let pair = pair?; @@ -226,18 +233,23 @@ pub(super) fn hints<'py>( claimed.set_item(lower(&entry.getattr("origin_column")?)?, py.None())?; claimed.set_item(lower(&entry.getattr("destination_column")?)?, py.None())?; } - native( + object( py, - NativeAttr::RenameHints, - &[ - ("model_name", model_reads::name_object(&candidate.model)?), - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("excluded", excluded.call_method1("union", (claimed,))?), - ( - "live_columns", - PyFrozenSet::new(py, candidate.warehouse.keys().iter())?.into_any(), - ), - ], + rename_hints( + py, + Some(keywords( + py, + &[ + ("model_name", model_reads::name_object(&candidate.model)?), + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", excluded.call_method1("union", (claimed,))?), + ( + "live_columns", + PyFrozenSet::new(py, candidate.warehouse.keys().iter())?.into_any(), + ), + ], + )?), + )?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index e7f00d1aa4..333e6dc940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -3,7 +3,9 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; -use crate::bindings::_helpers::records::ids::{NativeAttr, NativeItem, RecordId}; +use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::{NativeItem, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; @@ -54,13 +56,16 @@ pub(super) fn definition_hash<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::ModelDefinitionHash, - &[ - ("model_name", model_reads::name_object(model)?), - ("query_sql", model_reads::query_sql(model)?), - ("dialect", snapshot.getattr("column_dialect")?), - ], + model_definition_hash( + model.py(), + &model_reads::name_object(model)?.extract::()?, + &model_reads::query_sql(model)?.extract::()?, + snapshot + .getattr("column_dialect")? + .extract::>()? + .as_deref(), + )?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index f1743b77de..936d5c1fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; @@ -9,7 +10,9 @@ use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -67,15 +70,20 @@ pub(super) fn overlay<'py>( if !candidate.fingerprint.is_none() && !candidate.previous.is_none() && !candidate.current.is_none() - && native( + && object( py, - NativeAttr::RenamesExplainChange, - &[ - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("renames", renames.into_any()), - ("input_columns", inputs.clone()), - ], + renames_explain_change( + py, + Some(keywords( + py, + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("renames", renames.into_any()), + ("input_columns", inputs.clone()), + ], + )?), + )?, )? .is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs index fe781f4777..479e79c652 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs @@ -32,7 +32,7 @@ fn projections(shape: &Bound<'_, PyAny>) -> PyResult> { /// Unique automatic column renames whose full query change is explained. #[pyfunction] #[pyo3(signature = (**options))] -fn identical_renames<'py>( +pub(crate) fn identical_renames<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -89,7 +89,7 @@ fn identical_renames<'py>( /// Explicit migration hints for newly added columns resembling removed live columns. #[pyfunction] #[pyo3(signature = (**options))] -fn rename_hints<'py>( +pub(crate) fn rename_hints<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index d081e0df53..4a1d4e2c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -2,11 +2,13 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; use crate::bindings::_helpers::planner_migrations::models::{ CompatibilityCheck, MigrationCompatibilityResult, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -142,21 +144,24 @@ pub(super) fn check_migration_compatibility<'py>( ("column_dialect", check.column_dialect.clone()), ], )?; - let change = call_with( - ®istry::native(py, NativeAttr::DetectModelChanges)?, - &[ - ("model", model.clone()), - ("snapshot", snapshot), - ("sql_analysis_enabled", check.sql_analysis_enabled.clone()), - ( - "query_change_tracking", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "full_refresh", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ], + let change = detect_model_changes( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot), + ("sql_analysis_enabled", check.sql_analysis_enabled.clone()), + ( + "query_change_tracking", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "full_refresh", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?), )?; let schema = model.getattr("schema_entry")?; let mut has_columns = inferred_rows(&model)?.is_some_and(|rows| !rows.is_empty()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 41f8d72f48..b7e5b4ad56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -1,13 +1,12 @@ //! Model ordering, selection and metadata at the migration discovery boundary. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; +use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_order; @@ -78,13 +77,15 @@ pub(super) fn metadata<'py>( ) -> PyResult> { let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; - let functions = native( + let functions = object( py, - NativeAttr::BuildFunctionLocalHashes, - &[ - ("functions", project.getattr("functions")?), - ("dialect", project.getattr("sql_analysis_dialect")?), - ], + build_function_local_hashes( + &project.getattr("functions")?, + project + .getattr("sql_analysis_dialect")? + .extract::>()? + .as_deref(), + )?, )?; let identities = host::entry(py, UserCodeHost::BuildHookIdentities)? .call1((project.getattr("hook_functions")?,))?; @@ -100,13 +101,13 @@ pub(super) fn metadata<'py>( for item in models { result.set_item( item.getattr("name")?, - call_with( - ®istry::native(py, NativeAttr::BuildModelVersionIdentityMetadataJson)?, - &[ - ("model", item.clone()), - ("function_local_hashes", functions.clone()), - ("hook_version_hashes", hooks.clone().into_any()), - ], + object( + py, + build_model_version_identity_metadata_json( + item, + functions.extract::>>()?, + hooks.clone().into_any().extract::>>()?, + )?, )?, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 0ce873333a..44eaaf8b95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -12,7 +12,7 @@ use std::collections::BTreeMap; /// Hash normalized SQL and non-storage metadata through rename identities. #[pyfunction] #[pyo3(signature = (**options))] -fn build_migration_fingerprint( +pub(crate) fn build_migration_fingerprint( py: Python<'_>, options: Option>, ) -> PyResult> { @@ -52,7 +52,7 @@ fn build_migration_fingerprint( /// Every name whose rename identity can change the migration fingerprint. #[pyfunction] #[pyo3(signature = (*, query_sql, metadata_json))] -fn migration_fingerprint_ref_names<'py>( +pub(crate) fn migration_fingerprint_ref_names<'py>( py: Python<'py>, query_sql: &str, metadata_json: &str, @@ -73,7 +73,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Attach persisted migration identities through the existing cache boundary. #[pyfunction] #[pyo3(signature = (**options))] -fn with_migration_fingerprints<'py>( +pub(crate) fn with_migration_fingerprints<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index b59d161658..9b2da1d432 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -12,7 +12,7 @@ mod discovery_events; mod discovery_models; mod discovery_requests; mod errors; -mod fingerprint; +pub(crate) mod fingerprint; mod known_input_columns; mod matching; mod model_entries; @@ -25,10 +25,10 @@ mod old_name_entries; mod old_name_objects; pub(crate) mod old_name_planning; mod old_name_references; -mod old_names; +pub(crate) mod old_names; mod planning_boundary; mod planning_execution; mod planning_origin; -mod projections; +pub(crate) mod projections; pub(crate) mod registration; mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 2dcffdd0e0..74ae995674 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -2,9 +2,10 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_migrations::column_objects::{native, replace, text}; +use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; +use crate::bindings::_helpers::planner_migrations::column_objects::{replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; -use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -90,13 +91,13 @@ pub(super) fn equivalent<'py>( ("definition", model_reads::query_sql(model)?), ( "definition_hash", - native( + object( py, - NativeAttr::ComputeQueryHash, - &[ - ("query_sql", model_reads::query_sql(model)?), - ("dialect", dialect.clone()), - ], + compute_query_hash( + py, + &model_reads::query_sql(model)?.extract::()?, + dialect.extract::>()?.as_deref(), + )?, )?, ), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 8a9849da3f..093f5c8be9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -1,7 +1,9 @@ //! Model migration orchestration over Python-owned live inspection. use crate::bindings::_helpers::compile_driver::python_calls::call_with; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; use crate::bindings::_helpers::planner_migrations::models::{ DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, @@ -10,8 +12,9 @@ use crate::bindings::_helpers::planner_migrations::models::{ use crate::bindings::_helpers::planner_migrations::planning_origin::{ require_same_database, resolve_origin, }; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ @@ -223,17 +226,22 @@ fn finish<'py>( } else { super::model_overlay::overlay(phase, state, &entries, &handovers)? }; - let reference_renames = native( + let reference_renames = object( py, - NativeAttr::PlanReferenceRenames, - &[ - ("runtime", runtime), - ("scope", scope), - ("snapshot", snapshot), - ("state", state.clone()), - ("schemas", schemas.clone()), - ("planned", planned.into_any()), - ], + plan_reference_renames( + py, + Some(keywords( + py, + &[ + ("runtime", runtime), + ("scope", scope), + ("snapshot", snapshot), + ("state", state.clone()), + ("schemas", schemas.clone()), + ("planned", planned.into_any()), + ], + )?), + )?, )?; let mut fields = vec![("reference_renames", reference_renames)]; if !entries.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 0b3fb70282..6d9a45f2e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,14 +1,18 @@ //! Assemble new and resumed old-name view steps from recorded facts. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; use crate::bindings::_helpers::planner_migrations::old_name_objects::{ expires, location, relation, retention, stores_history, }; +use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; @@ -136,26 +140,31 @@ impl<'py> OldNamePlanner<'py> { tracked: Bound<'py, PyAny>, retention: &Bound<'py, PyAny>, ) -> PyResult> { - native( + object( self.runtime.py(), - NativeAttr::OldNameViewSkipReason, - &[ - ("origin", origin.clone()), - ("origin_tracked", tracked), - ("retention", retention.clone()), - ( - "models", - PyTuple::new(self.runtime.py(), &self.models)?.into_any(), - ), - ("views", self.views.clone()), - ], + old_name_view_skip_reason( + self.runtime.py(), + Some(keywords( + self.runtime.py(), + &[ + ("origin", origin.clone()), + ("origin_tracked", tracked), + ("retention", retention.clone()), + ( + "models", + PyTuple::new(self.runtime.py(), &self.models)?.into_any(), + ), + ("views", self.views.clone()), + ], + )?), + )?, ) } fn aliases(&self, item: &Bound<'py, PyAny>) -> PyResult> { let py = self.runtime.py(); if !stores_history(item)? { - return registry::native(py, NativeAttr::DeclaredColumnAliases)?.call1((item,)); + return Ok(declared_column_aliases(py, item)?.into_any()); } let mut aliases: Vec> = Vec::new(); for entry in self.columns.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 505092d9f9..10a26869ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -1,13 +1,15 @@ //! Old-name planning orchestration over existing live state inspection. use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; @@ -56,10 +58,9 @@ fn schemas<'py>( needed.add(schema?)?; } } - native( + object( py, - NativeAttr::OldNameSchemasToRead, - &[("snapshot", snapshot), ("needed", needed.into_any())], + old_name_schemas_to_read(py, &snapshot, &needed.into_any())?, ) } @@ -139,18 +140,23 @@ pub(crate) fn plan_old_name_views<'py>( } } let entries = PyTuple::new(py, entries)?.into_any(); - let claims = native( + let claims = object( py, - NativeAttr::OldNameClaimErrors, - &[ - ("selected", PyTuple::new(py, selected)?.into_any()), - ("snapshot", phase.snapshot.clone()), - ("views", views.clone()), - ( - "now", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, - ), - ], + old_name_claim_errors( + py, + Some(keywords( + py, + &[ + ("selected", PyTuple::new(py, selected)?.into_any()), + ("snapshot", phase.snapshot.clone()), + ("views", views.clone()), + ( + "now", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), + ], + )?), + )?, )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; warnings.extend(super::old_name_references::errors( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index 994f90033c..763428c7d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -1,11 +1,15 @@ //! Diagnose literal references to compatibility-only old names. +use crate::bindings::_helpers::analysis::explicit_references::hard_coded_relation_remedy; +use crate::bindings::_helpers::analysis::explicit_references::old_name_reference_message; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::relations::match_project_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; @@ -83,11 +87,7 @@ pub(super) fn errors<'py>( for literal in literals.try_iter()? { let literal = literal?; let relation = literal.getattr("relation")?; - let matched = native( - py, - NativeAttr::MatchProjectRelation, - &[("index", index.clone()), ("relation", relation.clone())], - )?; + let matched = object(py, match_project_relation(&index, &relation, None, None)?)?; if matched.is_none() || matched.getattr("compatibility_for")?.is_none() { continue; } @@ -98,26 +98,21 @@ pub(super) fn errors<'py>( parts.push(part.extract()?); } } - let message: String = native( + let message: String = object( py, - NativeAttr::OldNameReferenceMessage, - &[ - ("owner_label", literal.getattr("owner_label")?), - ("written", text(py, &parts.join("."))), - ("method", literal.getattr("method")?), - ("model_name", matched.getattr("compatibility_for")?), - ], - )? - .extract()?; - let remedy: String = native( - py, - NativeAttr::HardCodedRelationRemedy, - &[ - ("owner_kind", literal.getattr("owner_kind")?), - ("ref", matched.getattr("ref")?), - ], + old_name_reference_message( + &literal.getattr("owner_label")?.extract::()?, + &parts.join("."), + &literal.getattr("method")?.extract::()?, + &matched.getattr("compatibility_for")?.extract::()?, + ), )? .extract()?; + let remedy: String = hard_coded_relation_remedy( + &literal.getattr("owner_kind")?.extract::()?, + &matched.getattr("ref")?, + None, + )?; let path = literal .getattr("relative_path")? .str()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 8ebfdd6972..b6a5675b08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::planner_migrations::main::{ }; #[pyfunction] -fn declared_column_aliases<'py>( +pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { @@ -36,7 +36,7 @@ fn declared_column_aliases<'py>( #[pyfunction] #[pyo3(signature = (*, snapshot, needed))] -fn old_name_schemas_to_read<'py>( +pub(crate) fn old_name_schemas_to_read<'py>( py: Python<'py>, snapshot: &Bound<'py, PyAny>, needed: &Bound<'py, PyAny>, @@ -97,7 +97,7 @@ fn claim_message(model: &str, view: &Bound<'_, PyAny>, now: &Bound<'_, PyAny>) - #[pyfunction] #[pyo3(signature = (**options))] -fn old_name_claim_errors<'py>( +pub(crate) fn old_name_claim_errors<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -155,7 +155,7 @@ fn old_name_claim_errors<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn old_name_view_skip_reason<'py>( +pub(crate) fn old_name_view_skip_reason<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index a3398a3f48..97951f109f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -1,13 +1,13 @@ //! Assemble migration policy and redirected live cursor inspection requests. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::retention::effective_target_config; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_warehouse::snapshot::gather_redirected_cursor_snapshots; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; /// The target's policy for migration origins that no longer exist. @@ -52,34 +52,37 @@ pub(super) fn redirected_cursor_snapshots<'py>( &project, overrides.getattr("full_refresh")?.is_truthy()?, )?; - call_with( - ®istry::native(py, NativeAttr::GatherRedirectedCursorSnapshots)?, - &[ - ("project", project), - ( - "adapter", - runtime_field(runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(runtime, |runtime| &runtime.connection)?, - ), - ("existing_relations", existing_relations), - ("target_relations", target_relations), - ("cursor_scope", cursor_scope), - ("full_refresh_model_names", full_refresh), - ( - "deferred_locations", - phase.deferral.getattr("deferred_locations")?, - ), - ( - "on_progress", - runtime_field(runtime, |runtime| &runtime.on_progress)?, - ), - ( - "origin_cursor_columns", - origin_cursor_columns.unwrap_or_else(|| py.None().into_bound(py)), - ), - ], + gather_redirected_cursor_snapshots( + py, + Some(keywords( + py, + &[ + ("project", project), + ( + "adapter", + runtime_field(runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(runtime, |runtime| &runtime.connection)?, + ), + ("existing_relations", existing_relations), + ("target_relations", target_relations), + ("cursor_scope", cursor_scope), + ("full_refresh_model_names", full_refresh), + ( + "deferred_locations", + phase.deferral.getattr("deferred_locations")?, + ), + ( + "on_progress", + runtime_field(runtime, |runtime| &runtime.on_progress)?, + ), + ( + "origin_cursor_columns", + origin_cursor_columns.unwrap_or_else(|| py.None().into_bound(py)), + ), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs index cf135f6f12..cec042fd6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs @@ -36,7 +36,7 @@ fn read_shape(shape: &Bound<'_, PyAny>) -> PyResult { /// Named projections of one plain SELECT, or None when unknowable. #[pyfunction] #[pyo3(signature = (*, query_sql, dialect))] -fn parse_query_shape<'py>( +pub(crate) fn parse_query_shape<'py>( py: Python<'py>, query_sql: &str, dialect: Option<&str>, @@ -125,7 +125,10 @@ pub(crate) fn explain( /// Whether output renaming explains the whole query change. #[pyfunction] #[pyo3(signature = (**options))] -fn renames_explain_change(py: Python<'_>, options: Option>) -> PyResult { +pub(crate) fn renames_explain_change( + py: Python<'_>, + options: Option>, +) -> PyResult { let arguments = KeywordArguments::read( py, "renames_explain_change", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index 77f3b74721..14c1d73c87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -1,5 +1,7 @@ //! Read-only warehouse evidence gathered once for all model migrations in one plan. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ @@ -18,8 +20,6 @@ use crate::bindings::_helpers::planner_migrations::constants::{ GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; use crate::bindings::_helpers::state_rows::migration_objects::layout; @@ -175,8 +175,8 @@ impl MigrationStateInspection { .call_method1("get", (key,))?; let latest: Bound<'py, PyAny> = if known.is_none() { let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); - let fingerprint_set: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ReadLatestFingerprints)?, + let fingerprint_set: Bound<'py, PyAny> = read_latest_fingerprints(Some(&keywords( + py, &[ ("connection", self.connection.bind(py).clone()), ("execute", adapter.getattr("execute")?), @@ -192,7 +192,7 @@ impl MigrationStateInspection { adapter.getattr("render_read_latest_fingerprints_sql")?, ), ], - )?; + )?))?; let by_identity: Bound<'py, PyAny> = fingerprint_set.getattr("fingerprints_by_identity")?; if by_identity.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index b032188b7c..645b9cb43c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,13 +1,17 @@ //! Selected-scope buildability validation before change planning. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_identity::graph::build_execution_edge_origins; +use crate::bindings::_helpers::planner_identity::graph::check_buildability; +use crate::bindings::_helpers::planner_identity::graph::missing_upstream_message; use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; -use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -29,38 +33,35 @@ pub(crate) fn check_selected_scope_buildability<'py>( external_seeds.push(seed.getattr("key")?); } } - let missing = native( + let missing = object( py, - NativeAttr::CheckBuildability, - &[ - ("selected_keys", selected_scope.getattr("selected_keys")?), - ("upstream_deps", selected_scope.getattr("upstream_deps")?), - ( - "snapshot", - warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, - ), - ( - "deferred_relations", - deferral.getattr("deferred_relations")?, - ), - ("satisfied_keys", frozenset(py, &external_seeds)?), - ], + check_buildability( + py, + Some(keywords( + py, + &[ + ("selected_keys", selected_scope.getattr("selected_keys")?), + ("upstream_deps", selected_scope.getattr("upstream_deps")?), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), + ( + "deferred_relations", + deferral.getattr("deferred_relations")?, + ), + ("satisfied_keys", frozenset(py, &external_seeds)?), + ], + )?), + )?, )?; if missing.is_truthy()? { - let message: String = native( + let message: String = object( py, - NativeAttr::MissingUpstreamMessage, - &[ - ("missing", missing), - ( - "edge_origins", - native( - py, - NativeAttr::BuildExecutionEdgeOrigins, - &[("project", project.clone())], - )?, - ), - ], + missing_upstream_message( + &missing, + &object(py, build_execution_edge_origins(&project)?)?, + )?, )? .extract()?; return Err(planner_error( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index c4f6f2780e..76370f1dca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,18 +1,19 @@ //! Write-hash reconciliation and plan entry construction. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; use crate::bindings::_helpers::planner_entries::models::{ PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest, }; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_identity::identities::merge_recomputed_model_changes; +use crate::bindings::_helpers::planner_identity::identities::with_honest_model_write_hashes; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; -use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; @@ -24,29 +25,24 @@ pub(crate) fn reconcile_execution_changes<'py>( changes: &Bound<'py, PyAny>, ) -> PyResult> { let py = execution_scope.py(); - let honest = native( + let honest = with_honest_model_write_hashes( py, - NativeAttr::WithHonestModelWriteHashes, - &[ - ("scope", execution_scope.clone()), - ("snapshot", snapshot.clone()), - ("changes", changes.clone()), - ( - "version_identities", - class_field(identities, |i: &PlannerIdentityContext| { - &i.version_identities - })?, - ), - ], - )?; - let resolved = native( - py, - NativeAttr::MergeRecomputedModelChanges, - &[ - ("resolved_actions", resolved_actions.clone()), - ("changes", honest.clone()), - ], + Some(keywords( + py, + &[ + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), + ("changes", changes.clone()), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?, + ), + ], + )?), )?; + let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; Ok(ChangeReconciliation { changes: honest, resolved_actions: resolved, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 81a56d9ab4..1ee9b4fd85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::records::ids::{ErrorId, NativeItem, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; +use pyo3::{BoundObject, IntoPyObject}; /// A Python `str`. pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { @@ -181,3 +182,12 @@ pub(crate) fn model_key<'py>( ], ) } + +/// A native helper's Rust result as a Python object. +pub(crate) fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> +where + T: IntoPyObject<'py>, + PyErr: From, +{ + Ok(value.into_pyobject(py)?.into_bound().into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index c6783007b9..381a50a5a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,17 +2,22 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; +use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; +use crate::bindings::_helpers::planner_migrations::fingerprint::with_migration_fingerprints; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ ChangeReconciliation, ModelEntryResults, PlanOutputFields, }; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ native, replace, sorted_names, strings, text, tuple, }; +use crate::bindings::_helpers::planner_phases::retention::plan_retention; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; @@ -175,23 +180,30 @@ pub(crate) fn add_storage_policies<'py>( let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } - entries = native( + entries = object( py, - NativeAttr::WithMigrationFingerprints, - &[ - ("entries", entries), - ("models_by_name", models.into_any()), - ( - "fingerprints", - warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, - ), - ( - "dialect", - runtime - .getattr("adapter")? - .call_method0("sql_analysis_dialect")?, - ), - ], + with_migration_fingerprints( + py, + Some(keywords( + py, + &[ + ("entries", entries), + ("models_by_name", models.into_any()), + ( + "fingerprints", + warehouse_field(warehouse, |warehouse| { + &warehouse.migration_fingerprints + })?, + ), + ( + "dialect", + runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?, + ), + ], + )?), + )?, )?; } let entries = with_column_rename_hints( @@ -212,13 +224,16 @@ pub(crate) fn add_storage_policies<'py>( ], )?; let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let storage = [ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scope", selected), - ]; - let table_types = native(py, NativeAttr::PlanTableTypes, &storage)?; - let retention = native(py, NativeAttr::PlanRetention, &storage)?; + let table_types = native( + py, + NativeAttr::PlanTableTypes, + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scope", selected.clone()), + ], + )?; + let retention = plan_retention(runtime, warehouse, &selected)?; output.set("model_entries", entries); output.set("migration_entries", migration_entries); output.set("column_migration_entries", column_entries); @@ -250,39 +265,44 @@ pub(crate) fn add_plan_warnings<'py>( if !policies.getattr("selection_diagnostics")?.is_truthy()? { return Ok(output); } - let stale = native( + let stale = object( py, - NativeAttr::BuildStaleOutOfSelectionWarnings, - &[ - ( - "original_scope", - class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, - ), - ( - "execution_scope", - class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, - ), - ("changes", stale_warning_changes.clone()), - ( - "snapshot", - warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, - ), - ( - "version_identities", - class_field(identities, |i: &PlannerIdentityContext| { - &i.stale_warning_identities - })?, - ), - ("source_freshness", source_freshness.clone()), - ( - "include_sources", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "model_changes_complete", - PyBool::new(py, true).to_owned().into_any(), - ), - ], + build_stale_out_of_selection_warnings( + py, + Some(keywords( + py, + &[ + ( + "original_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + ), + ( + "execution_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + ), + ("changes", stale_warning_changes.clone()), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?, + ), + ("source_freshness", source_freshness.clone()), + ( + "include_sources", + PyBool::new(py, false).to_owned().into_any(), + ), + ( + "model_changes_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ], + )?), + )?, )?; if stale.is_truthy()? { let warnings = concat(py, &[output.field("warnings")?, stale])?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index e58fa52e5b..39a084effd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::constants::{ @@ -14,7 +15,7 @@ use crate::bindings::_helpers::planner_phases::objects::{ planner_model, replace, sorted_names, text, tuple, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -39,15 +40,7 @@ pub(crate) fn effective_target_config<'py>( if project_config.is_none() || local_config.is_none() || target_name.is_none() { return native(py, RecordId::TargetConfig, &[]); } - native( - py, - NativeAttr::ResolveTargetConfig, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name), - ], - ) + resolve_target_config(py, &project_config, &local_config, &target_name) } /// A deterministic identifier-fitted conversion copy name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs index 96c8604552..6259243ca5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs @@ -1,14 +1,13 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; #[pymethods] impl PlannerRuntime { @@ -97,9 +96,7 @@ impl PlannerWarehouseState { migration_fingerprints: if args.passed("migration_fingerprints")? { args.value("migration_fingerprints")?.unbind() } else { - registry::native(py, NativeAttr::MigrationFingerprintCache)? - .call0()? - .unbind() + Py::new(py, MigrationFingerprintCache::open(py, None)?)?.into_any() }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index b7dded0015..0fd177e226 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,10 +1,12 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; -use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use pyo3::IntoPyObject; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{replace, tuple}; +use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -18,47 +20,48 @@ pub(crate) fn resolve_planner_scopes<'py>( ) -> PyResult> { let py = project.py(); let auto_load = policies.getattr("auto_load_sources")?; - let selected = native( + let selected = build_planner_scope( py, - NativeAttr::BuildPlannerScope, - &[ - ("project", project.clone()), - ("select", selection.getattr("select")?), - ("exclude", selection.getattr("exclude")?), - ("auto_load_sources", auto_load.clone()), - ("selected_keys", selection.getattr("selected_keys")?), - ( - "python_read_source_names", - selection.getattr("python_read_source_names")?, - ), - ( - "sql_test_selection", - selection.getattr("sql_test_selection")?, - ), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ("auto_load_sources", auto_load.clone()), + ("selected_keys", selection.getattr("selected_keys")?), + ( + "python_read_source_names", + selection.getattr("python_read_source_names")?, + ), + ( + "sql_test_selection", + selection.getattr("sql_test_selection")?, + ), + ], + )?), )?; let empty = PyTuple::empty(py).into_any(); - let full = native( + let full = build_planner_scope( py, - NativeAttr::BuildPlannerScope, - &[ - ("project", project.clone()), - ("select", empty.clone()), - ("exclude", empty), - ("auto_load_sources", auto_load), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("select", empty.clone()), + ("exclude", empty), + ("auto_load_sources", auto_load), + ], + )?), )?; - let stale_keys = native( + let stale_keys = object( py, - NativeAttr::TransitiveClosureMany, - &[ - ("starts", selected.getattr("selected_keys")?), - ("edges", full.getattr("upstream_deps")?), - ( - "include_starts", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + transitive_closure_many( + &selected.getattr("selected_keys")?, + &full.getattr("upstream_deps")?, + true, + None, + )?, )?; let all_keys = PyDict::new(py); for (name, key) in full.getattr("all_keys")?.downcast::()?.iter() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index fa27998cc4..d0ef32f341 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -1,12 +1,14 @@ //! Snapshot conversion and cursor window policy boundary. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; use crate::bindings::_helpers::cursor::scalars::try_parse; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_resolution::maximum_safety::apply_maximum_start_policy; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -114,7 +116,7 @@ fn read_policy<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn apply_typed_cursor_replay_policy<'py>( +pub(crate) fn apply_typed_cursor_replay_policy<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -168,7 +170,7 @@ pub(super) fn required_scalar<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_cursor_bounds<'py>( +pub(crate) fn compute_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -257,10 +259,7 @@ fn compute_cursor_bounds<'py>( .into_any(), ), ]); - let replayed = call_with( - ®istry::native(py, NativeAttr::ApplyTypedCursorReplayPolicy)?, - &replay_args, - )?; + let replayed = apply_typed_cursor_replay_policy(py, Some(keywords(py, &replay_args)?))?; let bounds = call_with( &bounds_class, &[("start", replayed), ("end", session.python_scalar(&end)?)], @@ -292,10 +291,7 @@ fn compute_cursor_bounds<'py>( ] { policy_args.push((key, args.value(key)?)); } - call_with( - ®istry::native(py, NativeAttr::ApplyMaximumStartPolicy)?, - &policy_args, - ) + apply_maximum_start_policy(py, Some(keywords(py, &policy_args)?)) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index f28f68af41..897166f1a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,22 +1,33 @@ //! Model cursor orchestration over live warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; +use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; +use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_cursor_window; +use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_input_rebuild_refusal; +use crate::bindings::_helpers::planner_resolution::cursor_windows::unreadable_cursor_inputs_error; +use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; +use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{IntoPyObject, PyErr}; @@ -92,14 +103,14 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let cursor_type = config(&model, "cursor_type")?; let cursor_grain = config(&model, "cursor_grain")?; let (start_override, end_override) = request.cursor_overrides.clone(); - let bounded = call_with( - ®istry::native(py, NativeAttr::ResolveBoundedCursorOverride)?, - &[ - ("start_cursor_override", start_override.clone()), - ("end_cursor_override", end_override.clone()), - ("cursor_type", cursor_type.clone()), - ("cursor_grain", cursor_grain.clone()), - ], + let bounded = object( + py, + resolve_bounded_cursor_override( + &start_override, + &end_override, + &cursor_type, + &cursor_grain, + )?, )?; if !bounded.is_none() { return Ok(bounded); @@ -108,10 +119,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? .call_method1("get", (model_reads::name_object(&model)?,))?; - let roles = call_with( - ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, - &[("model", model.clone())], - )?; + let roles = resolve_cursor_input_roles(&model)?; let producers = class_field(context, |c: &ModelPlanContext| { &c.runtime_cursor_producer_names })?; @@ -125,23 +133,19 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let full_refresh = request.full_refresh; let replaces = request.replaces_relation; if !full_refresh && !runtime_owned && !cursor_snapshot.is_none() { - let empty = call_with( - ®istry::native(py, NativeAttr::CursorInputsWithoutRows)?, - &[("cursor_snapshot", cursor_snapshot.clone())], - )? - .getattr("input_names")?; + let empty = cursor_inputs_without_rows(&cursor_snapshot)?.getattr("input_names")?; if empty.is_truthy()? && replaces && (destination_exists(&model, snapshot)? || history_redirected(&model, &cursor_snapshot)?) { - let refusal = call_with( - ®istry::native(py, NativeAttr::EmptyInputRebuildRefusal)?, - &[ - ("model_name", model_reads::name_object(&model)?), - ("input_names", empty.clone()), - ("cursor_type", cursor_type.clone()), - ], + let refusal = object( + py, + empty_input_rebuild_refusal( + &model_reads::name_object(&model)?.extract::()?, + empty.extract::>()?, + cursor_type.extract::>()?.as_deref(), + ), )?; return Err(PyErr::from_value(call_with( ®istry::error(py, ErrorId::PlannerInput)?, @@ -153,13 +157,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( )?)); } if empty.is_truthy()? { - return call_with( - ®istry::native(py, NativeAttr::EmptyCursorWindow)?, - &[ - ("cursor_type", cursor_type), - ("cursor_start", config(&model, "cursor_start")?), - ], - ); + return empty_cursor_window(&cursor_type, &config(&model, "cursor_start")?); } if !cursor_snapshot .getattr("watermarks_available")? @@ -171,13 +169,11 @@ pub(crate) fn compute_model_cursor_bounds<'py>( } else { cursor_snapshot.getattr("unavailable_watermark_tags")? }; - return Err(PyErr::from_value(call_with( - ®istry::native(py, NativeAttr::UnreadableCursorInputsError)?, - &[ - ("model_name", model_reads::name_object(&model)?), - ("cursor_type", cursor_type), - ("input_names", names), - ], + return Err(PyErr::from_value(unreadable_cursor_inputs_error( + py, + &model_reads::name_object(&model)?.extract::()?, + cursor_type.extract::>()?.as_deref(), + names.extract::>()?, )?)); } } @@ -191,10 +187,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(py.None().into_bound(py)); } if replaces { - cursor_snapshot = call_with( - ®istry::native(py, NativeAttr::WithoutDestinationCursor)?, - &[("cursor_snapshot", cursor_snapshot)], - )?; + cursor_snapshot = without_destination_cursor(&cursor_snapshot)?; } let backfill = &request.backfill; let duration = if backfill.getattr("action")?.eq("bounded")? { @@ -217,46 +210,52 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ("incremental_mode", incremental_mode), ], )?; - let bounds = call_with( - ®istry::native(py, NativeAttr::ComputeCursorBounds)?, - &[ - ("cursor_snapshot", cursor_snapshot.clone()), - ("cursor_type", cursor_type.clone()), - ("cursor_start", config(&model, "cursor_start")?), - ("lookback", config(&model, "lookback")?), - ("backfill_duration", duration), - ("start_cursor_override", start_override.clone()), - ("end_cursor_override", end_override.clone()), - ( - "is_microbatch", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ("cursor_grain", cursor_grain.clone()), - ("maximum_start_policy", policy), - ], + let bounds = compute_cursor_bounds( + py, + Some(keywords( + py, + &[ + ("cursor_snapshot", cursor_snapshot.clone()), + ("cursor_type", cursor_type.clone()), + ("cursor_start", config(&model, "cursor_start")?), + ("lookback", config(&model, "lookback")?), + ("backfill_duration", duration), + ("start_cursor_override", start_override.clone()), + ("end_cursor_override", end_override.clone()), + ( + "is_microbatch", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ("cursor_grain", cursor_grain.clone()), + ("maximum_start_policy", policy), + ], + )?), )?; if bounds.is_none() { return Ok(bounds); } - call_with( - ®istry::native(py, NativeAttr::ResolveFutureCursorSafety)?, - &[ - ("bounds", bounds), - ("cursor_type", cursor_type), - ("cursor_grain", cursor_grain), - ("config", future_config), - ( - "invocation_time", - class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, - ), - ( - "has_complete_override", - (!start_override.is_none() && !end_override.is_none()) - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ("input_evidence", cursor_snapshot.getattr("input_evidence")?), - ], + resolve_future_cursor_safety( + py, + Some(keywords( + py, + &[ + ("bounds", bounds), + ("cursor_type", cursor_type), + ("cursor_grain", cursor_grain), + ("config", future_config), + ( + "invocation_time", + class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, + ), + ( + "has_complete_override", + (!start_override.is_none() && !end_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ("input_evidence", cursor_snapshot.getattr("input_evidence")?), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs index 8d61ea65dc..c066e4a339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -93,7 +93,7 @@ fn resolve_future_cursor_config<'py>( /// Resolve automatic-start policy with model config taking precedence. #[pyfunction] #[pyo3(signature = (*, model, project_config))] -fn resolve_start_cursor_config<'py>( +pub(crate) fn resolve_start_cursor_config<'py>( model: &Bound<'py, PyAny>, project_config: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index 2916eae334..e765b7856f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -1,24 +1,35 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::{parse_value, sentinel_to_token}; use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; +use crate::bindings::_helpers::model_config::config_values::config_cursor_bound; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; +use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; +use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; +use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; @@ -67,14 +78,6 @@ pub(crate) struct CursorWindowInputs<'py> { pub(crate) cursor_input_relations: Bound<'py, PyAny>, } -fn native<'py>( - py: Python<'py>, - name: NativeAttr, - pairs: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - call_with(®istry::native(py, name)?, pairs) -} - /// One string-valued model configuration entry. pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { Ok( @@ -88,13 +91,9 @@ pub(crate) fn config_bound<'py>( model: &Bound<'py, PyAny>, key: &str, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::GetConfigCursorBound, - &[ - ("values", model.getattr("config")?.getattr("values")?), - ("key", string(model.py(), key)), - ], + config_cursor_bound(&model.getattr("config")?.getattr("values")?, key)?, ) } @@ -184,10 +183,9 @@ fn rolling_window_bounds<'py>( ) -> PyResult> { let py = model.py(); let grain = config_str(model, "cursor_grain")?; - let lookback = native( + let lookback = object( py, - NativeAttr::ResolveMicrobatchLookback, - &[("model", model.clone()), ("effective_grain", grain.clone())], + resolve_microbatch_lookback(model, grain.extract::>()?)?, )?; let lookback = if lookback.is_truthy()? { lookback @@ -255,24 +253,26 @@ fn apply_future_safety<'py>( if bounds.is_none() || bounds.getattr("end")?.eq(sentinel(py, "END")?)? { return Ok(bounds); } - native( + resolve_future_cursor_safety( py, - NativeAttr::ResolveFutureCursorSafety, - &[ - ("bounds", bounds), - ("cursor_type", typing.0.clone()), - ("cursor_grain", typing.1.clone()), - ("config", inputs.policies.0.clone()), - ("invocation_time", inputs.invocation_time.clone()), - ( - "has_complete_override", - boolean( - py, - !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), - )?, - ), - ("input_evidence", input_evidence), - ], + Some(keywords( + py, + &[ + ("bounds", bounds), + ("cursor_type", typing.0.clone()), + ("cursor_grain", typing.1.clone()), + ("config", inputs.policies.0.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "has_complete_override", + boolean( + py, + !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), + )?, + ), + ("input_evidence", input_evidence), + ], + )?), ) } @@ -280,15 +280,14 @@ fn bounded_override<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::ResolveBoundedCursorOverride, - &[ - ("start_cursor_override", inputs.overrides.0.clone()), - ("end_cursor_override", inputs.overrides.1.clone()), - ("cursor_type", config_str(model, "cursor_type")?), - ("cursor_grain", config_str(model, "cursor_grain")?), - ], + resolve_bounded_cursor_override( + &inputs.overrides.0.clone(), + &inputs.overrides.1.clone(), + &config_str(model, "cursor_type")?, + &config_str(model, "cursor_grain")?, + )?, ) } @@ -325,21 +324,23 @@ fn snapshot_bounds<'py>( ("incremental_mode", config_str(model, "incremental_mode")?), ], )?; - let bounds = native( + let bounds = compute_cursor_bounds( py, - NativeAttr::ComputeCursorBounds, - &[ - ("cursor_snapshot", request.cursor_snapshot.clone()), - ("cursor_type", request.typing.0.clone()), - ("cursor_start", config_bound(model, "cursor_start")?), - ("lookback", request.lookback), - ("backfill_duration", backfill_duration(&inputs.backfill)?), - ("start_cursor_override", inputs.overrides.0.clone()), - ("end_cursor_override", inputs.overrides.1.clone()), - ("is_microbatch", boolean(py, request.is_microbatch)?), - ("cursor_grain", request.typing.1.clone()), - ("maximum_start_policy", policy), - ], + Some(keywords( + py, + &[ + ("cursor_snapshot", request.cursor_snapshot.clone()), + ("cursor_type", request.typing.0.clone()), + ("cursor_start", config_bound(model, "cursor_start")?), + ("lookback", request.lookback), + ("backfill_duration", backfill_duration(&inputs.backfill)?), + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("is_microbatch", boolean(py, request.is_microbatch)?), + ("cursor_grain", request.typing.1.clone()), + ("maximum_start_policy", policy), + ], + )?), )?; let bounds = apply_future_safety( bounds, @@ -360,11 +361,7 @@ fn model_cursor_snapshot<'py>( } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { - native( - cursor_snapshot.py(), - NativeAttr::WithoutDestinationCursor, - &[("cursor_snapshot", cursor_snapshot)], - ) + without_destination_cursor(&cursor_snapshot) } /// Compute cursor bounds for inclusion on the plan entry. @@ -449,34 +446,19 @@ pub(crate) fn microbatch_range<'py>( for relation in inputs.cursor_input_relations.try_iter()? { grains.push(relation?.getattr("cursor_grain")?); } - let effective_grain = native( - py, - NativeAttr::ResolveEffectiveTimestampGrain, - &[ - ("cursor_type", cursor_type.clone()), - ("downstream_grain", config_str(model, "cursor_grain")?), - ("microbatch_strategy", strategy), - ("cursor_input_grains", PyTuple::new(py, grains)?.into_any()), - ], + let effective_grain = resolve_effective_timestamp_grain( + &cursor_type, + &config_str(model, "cursor_grain")?, + &PyTuple::new(py, grains)?.into_any(), + strategy.extract::>()?.as_deref(), )?; if cursor_type.eq(TIMESTAMP)? && !effective_grain.is_none() { - cursor_snapshot = native( - py, - NativeAttr::NormalizeCursorSnapshotGrain, - &[ - ("cursor_snapshot", cursor_snapshot), - ("cursor_type", cursor_type.clone()), - ("effective_grain", effective_grain.clone()), - ], - )?; + cursor_snapshot = + normalize_cursor_snapshot_grain(&cursor_snapshot, &cursor_type, &effective_grain)?; } - let lookback = native( + let lookback = object( py, - NativeAttr::ResolveMicrobatchLookback, - &[ - ("model", model.clone()), - ("effective_grain", effective_grain.clone()), - ], + resolve_microbatch_lookback(model, effective_grain.extract::>()?)?, )?; snapshot_bounds( model, @@ -511,11 +493,7 @@ pub(crate) fn empty_cursor_inputs<'py>( if cursor_snapshot.is_none() { return nothing(); } - let decision = native( - py, - NativeAttr::CursorInputsWithoutRows, - &[("cursor_snapshot", cursor_snapshot)], - )?; + let decision = cursor_inputs_without_rows(&cursor_snapshot)?; Ok(( decision.getattr("input_names")?, decision.getattr("waiting_on_empty_inputs")?.is_truthy()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index 096a31951d..493912ad2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -41,7 +41,7 @@ fn decide_empty_cursor_inputs<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot))] -fn cursor_inputs_without_rows<'py>( +pub(crate) fn cursor_inputs_without_rows<'py>( cursor_snapshot: &Bound<'py, PyAny>, ) -> PyResult> { if cursor_snapshot @@ -74,7 +74,7 @@ fn cursor_inputs_without_rows<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type, cursor_start))] -fn empty_cursor_window<'py>( +pub(crate) fn empty_cursor_window<'py>( cursor_type: &Bound<'py, PyAny>, cursor_start: &Bound<'py, PyAny>, ) -> PyResult> { @@ -117,7 +117,7 @@ fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { #[pyfunction] #[pyo3(signature = (*, model_name, input_names, cursor_type))] -fn empty_input_rebuild_refusal( +pub(crate) fn empty_input_rebuild_refusal( model_name: &str, input_names: Vec, cursor_type: Option<&str>, @@ -136,7 +136,7 @@ fn empty_input_rebuild_refusal( #[pyfunction] #[pyo3(signature = (*, model_name, cursor_type, input_names))] -fn unreadable_cursor_inputs_error<'py>( +pub(crate) fn unreadable_cursor_inputs_error<'py>( py: Python<'py>, model_name: &str, cursor_type: Option<&str>, @@ -172,7 +172,7 @@ fn unreadable_cursor_inputs_error<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot))] -fn without_destination_cursor<'py>( +pub(crate) fn without_destination_cursor<'py>( cursor_snapshot: &Bound<'py, PyAny>, ) -> PyResult> { let py = cursor_snapshot.py(); @@ -185,7 +185,7 @@ fn without_destination_cursor<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] -fn resolve_effective_timestamp_grain<'py>( +pub(crate) fn resolve_effective_timestamp_grain<'py>( cursor_type: &Bound<'py, PyAny>, downstream_grain: &Bound<'py, PyAny>, cursor_input_grains: &Bound<'py, PyAny>, @@ -220,7 +220,7 @@ fn resolve_effective_timestamp_grain<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot, cursor_type, effective_grain))] -fn normalize_cursor_snapshot_grain<'py>( +pub(crate) fn normalize_cursor_snapshot_grain<'py>( cursor_snapshot: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, effective_grain: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs index 81ac8cc755..67d3aa58c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -18,7 +18,7 @@ use std::cmp::Ordering; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_future_cursor_safety<'py>( +pub(crate) fn resolve_future_cursor_safety<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs index e2b3f6c066..40c12926b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -14,17 +14,21 @@ fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { } #[pyfunction] -fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_model_locations<'py>( + models: &Bound<'py, PyAny>, +) -> PyResult> { locations(models) } #[pyfunction] -fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { locations(seeds) } #[pyfunction] -fn build_function_locations<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_function_locations<'py>( + functions: &Bound<'py, PyAny>, +) -> PyResult> { locations(functions) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 39b8a0efe3..7b2321512b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -1,7 +1,11 @@ //! Maximum automatic-start recovery and evidence boundary. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ optional_scalar, required_scalar, scalars, }; @@ -9,7 +13,7 @@ use crate::bindings::_helpers::planner_resolution::cursor_safety::{ horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -23,7 +27,7 @@ use std::cmp::Ordering; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn maximum_allowed_start<'py>( +pub(crate) fn maximum_allowed_start<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult { @@ -64,7 +68,7 @@ fn maximum_allowed_start<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn apply_maximum_start_policy<'py>( +pub(crate) fn apply_maximum_start_policy<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -171,10 +175,7 @@ fn apply_maximum_start_policy<'py>( ] { replay.push((key, args.value(key)?)); } - call_with( - ®istry::native(py, NativeAttr::ApplyTypedCursorReplayPolicy)?, - &replay, - )? + apply_typed_cursor_replay_policy(py, Some(keywords(py, &replay)?))? } else { bounds.getattr("start")? }; @@ -247,11 +248,7 @@ fn apply_maximum_start_policy<'py>( } fn rendered(value: &Bound<'_, PyAny>) -> PyResult { - call_with( - ®istry::native(value.py(), NativeAttr::CursorRender)?, - &[("value", value.clone())], - )? - .extract() + object(value.py(), render_value(value)?)?.extract() } /// The message of an automatic start that exceeds its horizon. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index 063cba97d8..93bab4d84b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,20 +1,20 @@ //! Planner reference resolution with Python adapter rendering callbacks. mod constants; -mod cursor_bounds; +pub(crate) mod cursor_bounds; pub(crate) mod cursor_model; pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; mod cursor_safety; pub(crate) mod cursor_warnings; -mod cursor_windows; -mod future_safety; -mod locations; -mod maximum_safety; +pub(crate) mod cursor_windows; +pub(crate) mod future_safety; +pub(crate) mod locations; +pub(crate) mod maximum_safety; pub(crate) mod models; pub(crate) mod orchestration; -mod refs; +pub(crate) mod refs; pub(crate) mod registration; -mod sources; +pub(crate) mod sources; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index d27886200c..f1860f511b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -281,7 +281,7 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_function_sql<'py>( +pub(crate) fn resolve_function_sql<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index a614a1269d..bf84bd6ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -7,8 +7,7 @@ use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use crate::bindings::_helpers::planner_resolution::values::{ cursor_wrap, filtering_of, method, string, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::code_scanner::SqlCodeScanner; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -137,8 +136,7 @@ fn resolve_function_calls( if spans.is_empty() { return Ok(query_sql.to_owned()); } - let scanner_type = registry::native(py, NativeAttr::SqlCodeScanner)?; - let scan = scanner_type.call1((query_sql, py.None(), "SQL reference resolution"))?; + let scan = SqlCodeScanner::new(query_sql, None, "SQL reference resolution".to_owned()); let mut output = String::new(); let mut last = 0; let mut code_index = 0; @@ -146,9 +144,7 @@ fn resolve_function_calls( if table { let start = query_sql[..span.start].chars().count(); while code_index < start { - let end: Option = scan - .call_method1("non_code_end", (code_index,))? - .extract()?; + let end: Option = scan.non_code_end(py, code_index)?; code_index = end.unwrap_or(code_index + 1); } if code_index > start { @@ -175,10 +171,8 @@ fn resolve_function_calls( } else { "SQL UDF call" }; - let scanner = scanner_type.call1((query_sql, py.None(), context))?; - let end: usize = scanner - .call_method1("matching_paren", (query_sql[..suffix].chars().count(),))? - .extract()?; + let scanner = SqlCodeScanner::new(query_sql, None, context.to_owned()); + let end: usize = scanner.matching_paren(py, query_sql[..suffix].chars().count())?; let end = query_sql .char_indices() .nth(end + 1) @@ -203,9 +197,7 @@ fn resolve_function_calls( if table { let length = query_sql.chars().count(); while code_index < length { - let end: Option = scan - .call_method1("non_code_end", (code_index,))? - .extract()?; + let end: Option = scan.non_code_end(py, code_index)?; code_index = end.unwrap_or(code_index + 1); } } @@ -235,7 +227,7 @@ pub(crate) fn resolve_table_function_references( #[pyfunction] #[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] -fn apply_deferred_locations<'py>( +pub(crate) fn apply_deferred_locations<'py>( model_locations: &Bound<'py, PyDict>, seed_locations: &Bound<'py, PyDict>, deferred_locations: &Bound<'py, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 7e9c530b0e..4b0f31535e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -241,7 +241,7 @@ fn cast_source( #[pyfunction] #[pyo3(signature = (*, adapter, source_entry))] -fn render_source_read_relation( +pub(crate) fn render_source_read_relation( adapter: &Bound<'_, PyAny>, source_entry: &Bound<'_, PyAny>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index d22cd1591b..ea4175949b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,13 +1,18 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; +use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; +use crate::bindings::_helpers::planner_scenario::execution::build_scenario_execution_plan; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; use crate::bindings::_helpers::planner_scenario::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -38,16 +43,22 @@ pub(crate) fn build_cli_scenario_plan<'py>( let adapter = args.value("adapter")?; let (graph, warnings) = plan_scenario_graph(&scenario, &project, true, None)?; raise_for_warnings(warnings.as_any())?; - let hashes = call_with( - ®istry::native(py, NativeAttr::BuildScenarioHashIndex)?, - &[ - ("project_name", args.value("project_name")?), - ("scenarios", project.getattr("sql_scenarios")?), - ( - "run_namespace", - project.getattr("scenario")?.getattr("run_namespace")?, - ), - ], + let hashes = object( + py, + build_scenario_hash_index( + py, + Some(keywords( + py, + &[ + ("project_name", args.value("project_name")?), + ("scenarios", project.getattr("sql_scenarios")?), + ( + "run_namespace", + project.getattr("scenario")?.getattr("run_namespace")?, + ), + ], + )?), + )?, )?; let mut identities: Vec> = Vec::new(); for (field, kind) in [ @@ -70,17 +81,20 @@ pub(crate) fn build_cli_scenario_plan<'py>( )?); } } - let map = call_with( - ®istry::native(py, NativeAttr::BuildScenarioRelationMap)?, - &[ - ("scenario_name", scenario.getattr("name")?), - ("hash_prefix", hashes.get_item(scenario.getattr("name")?)?), - ("artifacts", PyTuple::new(py, identities)?.into_any()), - ( - "identifier_limit", - adapter.call_method0("maximum_identifier_length")?, - ), - ], + let map = build_scenario_relation_map( + py, + Some(keywords( + py, + &[ + ("scenario_name", scenario.getattr("name")?), + ("hash_prefix", hashes.get_item(scenario.getattr("name")?)?), + ("artifacts", PyTuple::new(py, identities)?.into_any()), + ( + "identifier_limit", + adapter.call_method0("maximum_identifier_length")?, + ), + ], + )?), )?; let (database, schema) = namespace(&project, &graph)?; let relations = build_scenario_relation_plan(&ScenarioRelations { @@ -91,19 +105,22 @@ pub(crate) fn build_cli_scenario_plan<'py>( database, schema, })?; - let result = call_with( - ®istry::native(py, NativeAttr::BuildScenarioExecutionPlan)?, - &[ - ("scenario", scenario), - ("project", project), - ("adapter", adapter), - ("graph_plan", graph), - ("relation_plan", relations), - ( - "source_lexical_syntax", - args.value("source_lexical_syntax")?, - ), - ], + let result = build_scenario_execution_plan( + py, + Some(keywords( + py, + &[ + ("scenario", scenario), + ("project", project), + ("adapter", adapter), + ("graph_plan", graph), + ("relation_plan", relations), + ( + "source_lexical_syntax", + args.value("source_lexical_syntax")?, + ), + ], + )?), )?; raise_for_warnings(&result.get_item(1)?)?; result.get_item(0) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index ac6fa6bdaf..ee04b3e7fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -1,14 +1,19 @@ //! Existing seed and function plan dataclass construction. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, sql::required_target, values::error, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::PyString; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; pub(super) fn seed_entries<'py>( @@ -51,10 +56,7 @@ pub(super) fn seed_entries<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - registry::native(py, NativeAttr::ExtractSeedColumns)?.call1((&seed,))?, - ), + ("columns", extract_seed_columns(&seed)?.into_any()), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -72,27 +74,30 @@ pub(super) fn function_entry<'py>( warehouse_columns: &Bound<'py, PyAny>, ) -> PyResult> { let py = function.py(); - let body = call_with( - ®istry::native(py, NativeAttr::ResolveFunctionSql)?, - &[ - ("adapter", context.adapter.clone()), - ("function", function.clone()), - ( - "model_locations", - context.relations.getattr("model_locations")?, - ), - ( - "seed_locations", - context.relations.getattr("seed_locations")?, - ), - ("function_locations", locations.clone()), - ("source_map", context.relations.getattr("source_map")?), - ("source_warehouse_columns", warehouse_columns.clone()), - ( - "star_exclude_keyword", - context.adapter.call_method0("star_exclude_keyword")?, - ), - ], + let body = resolve_function_sql( + py, + Some(keywords( + py, + &[ + ("adapter", context.adapter.clone()), + ("function", function.clone()), + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", warehouse_columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?), )?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); for name in [ @@ -108,8 +113,7 @@ pub(super) fn function_entry<'py>( fields.push(("body_sql", body)); fields.push(( "fingerprint_query_sql", - registry::native(py, NativeAttr::BuildCompiledFunctionFingerprintSql)? - .call1((function,))?, + PyString::new(py, &build_compiled_function_fingerprint_sql(function)?).into_any(), )); for name in [ "fingerprint_destination", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index c96c3b41d7..8c52b0241b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,11 +1,18 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::compiled_project::records::compiled_model_record; +use crate::bindings::_helpers::planner_entries::model_entries::plan_model; use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; +use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; use crate::bindings::_helpers::planner_scenario::{ entries::{function_entry, seed_entries}, @@ -16,7 +23,7 @@ use crate::bindings::_helpers::planner_scenario::{ }; use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -76,14 +83,9 @@ pub(crate) fn build_scenario_execution_plan<'py>( }; let models = by_attribute(&project.getattr("models")?, "name")?; let functions = by_attribute(&project.getattr("functions")?, "key")?; - let locations = registry::native(py, NativeAttr::BuildFunctionLocations)? - .call1((project.getattr("functions")?,))?; - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((&project,))?; - let ordered = call_with( - ®istry::native(py, NativeAttr::TopologicallyOrderKeys)?, - &[("upstream", upstream)], - )?; + let locations = build_function_locations(&project.getattr("functions")?)?.into_any(); + let upstream = build_execution_upstream_deps(&project)?.into_any(); + let ordered = object(py, topologically_order_keys(&upstream, None)?)?; let mut model_keys: Vec> = Vec::new(); for key in ordered.try_iter()? { let key = key?; @@ -127,33 +129,39 @@ pub(crate) fn build_scenario_execution_plan<'py>( &model_reads::query_sql(&model)?.extract::()?, &context.relations, )?; - let record = registry::native(py, NativeAttr::CompiledModelRecord)?.call1((&model,))?; + let record = compiled_model_record(&model)?; let changes = PyDict::new(py); changes.set_item("destination", destination)?; changes.set_item("query_sql", query)?; let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; let model_context = model_context(&project, &context, (&models, &locations, &columns))?; - let result = call_with( - ®istry::native(py, NativeAttr::PlanModel)?, - &[ - ("model", model), - ("snapshot", snapshot.clone()), - ("adapter", context.adapter.clone()), - ("context", model_context), - ( - "sql_analysis_enabled", - PyBool::new(py, analysis).to_owned().into_any(), - ), - ( - "query_change_tracking", - PyBool::new(py, false).to_owned().into_any(), - ), - ("full_refresh", PyBool::new(py, true).to_owned().into_any()), - ( - "cursor_overrides", - registry::class(py, RecordId::CursorOverridePair)?.call0()?, - ), - ], + let result = object( + py, + plan_model( + py, + Some(keywords( + py, + &[ + ("model", model), + ("snapshot", snapshot.clone()), + ("adapter", context.adapter.clone()), + ("context", model_context), + ( + "sql_analysis_enabled", + PyBool::new(py, analysis).to_owned().into_any(), + ), + ( + "query_change_tracking", + PyBool::new(py, false).to_owned().into_any(), + ), + ("full_refresh", PyBool::new(py, true).to_owned().into_any()), + ( + "cursor_overrides", + registry::class(py, RecordId::CursorOverridePair)?.call0()?, + ), + ], + )?), + )?, )?; entries.push(result.get_item(0)?); for warning in result.get_item(1)?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index c928d53cfa..66d16970e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod artifacts; mod cli; mod constants; mod entries; -mod execution; +pub(crate) mod execution; mod fixtures; mod graph; mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index 32528173d4..797ed05898 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -1,6 +1,7 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, }; @@ -8,8 +9,7 @@ use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, values::{error, string}, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; @@ -46,7 +46,7 @@ pub(super) fn resolve_sql( result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { let target = locations.call_method1("get", (name,))?; let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { - call_with(®istry::native(py, NativeAttr::RenderSourceRelation)?, &[("entry", target), ("adapter", context.adapter.clone())])? + object(py, render_source_relation(&target, Some(&context.adapter.clone()).filter(|value| !value.is_none()))?)? } else {target.getattr("qualified_name")?}; if target.is_none() { let marker = string(py, marker).repr()?.to_str()?.to_owned(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs index e7194181c9..3ad6d7b82e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -1,7 +1,7 @@ //! Planner scope Python boundary models. -mod loaders; +pub(crate) mod loaders; pub(crate) mod registration; -mod scope; -mod sources; +pub(crate) mod scope; +pub(crate) mod sources; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index a06abf06ca..769f3ff02d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -119,7 +119,9 @@ fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult(raw: &Bound<'py, PyString>) -> PyResult> { +pub(crate) fn parse_project_selector<'py>( + raw: &Bound<'py, PyString>, +) -> PyResult> { let carrier: Carrier<'py> = Carrier::single(raw)?; let parsed: ParsedSelector = parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; @@ -129,7 +131,7 @@ fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult( +pub(crate) fn resolve_graph_selection<'py>( graph: PyRef<'py, NativeProjectGraph>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, @@ -140,7 +142,7 @@ fn resolve_graph_selection<'py>( /// `--select` minus `--exclude` on a project graph, adding the functions selected models need. #[pyfunction] #[pyo3(signature = (*, graph, select, exclude))] -fn select_project_graph<'py>( +pub(crate) fn select_project_graph<'py>( graph: &Bound<'py, PyAny>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, @@ -234,7 +236,7 @@ fn match_project_selector<'py>( /// The upstream key closure of one compiled graph key. #[pyfunction] #[pyo3(signature = (*, key, upstream_deps))] -fn expand_project_upstream_keys<'py>( +pub(crate) fn expand_project_upstream_keys<'py>( key: Bound<'py, PyAny>, upstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index b420ee0868..d0c203f978 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -164,7 +164,7 @@ fn split_unit_test_selectors<'py>( /// Raise when a selector names a unit test; return the unchanged resource selectors. #[pyfunction] #[pyo3(signature = (*, select, exclude, discovered_inputs))] -fn reject_unit_test_selectors<'py>( +pub(crate) fn reject_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 4e5f95a646..7f917b0b88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -1,5 +1,10 @@ //! Assemble audit plan dataclasses using native rendering and scheduling. +use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; +use crate::bindings::_helpers::auditing::render::render_audit_sql; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::views::project_lineage_views; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::audits::{ reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, }; @@ -16,7 +21,7 @@ use crate::bindings::_helpers::planner_strategy::constants::{ AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, }; use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Render and validate one audit query with the supplied adapter callbacks. @@ -26,22 +31,22 @@ fn render_query<'py>( context: &str, ) -> PyResult> { let py = sql.py(); - let resolved = call_with( - ®istry::native(py, NativeAttr::RenderAuditSql)?, - &[ - ("unresolved_sql", sql), - ("model_locations", request.model_locations.clone()), - ("seed_locations", request.seed_locations.clone()), - ("source_map", request.source_map.clone()), - ("adapter", request.adapter.clone()), - ], + let resolved = object( + py, + render_audit_sql(Some(&keywords( + py, + &[ + ("unresolved_sql", sql), + ("model_locations", request.model_locations.clone()), + ("seed_locations", request.seed_locations.clone()), + ("source_map", request.source_map.clone()), + ("adapter", request.adapter.clone()), + ], + )?))?, )?; - call_with( - ®istry::native(py, NativeAttr::AssertNoUnresolvedSqlMarkers)?, - &[ - ("sql", resolved.clone()), - ("context", PyString::new(py, context).into_any()), - ], + object( + py, + assert_no_unresolved_sql_markers(py, resolved.downcast::()?, context)?, )?; Ok(resolved) } @@ -230,7 +235,7 @@ pub(crate) fn build_selected_audit_entries<'py>( if selected.is_empty() { return Ok(entries); } - let lineage = registry::native(py, NativeAttr::ProjectLineageViews)?.call1((project,))?; + let lineage = project_lineage_views(py, &project)?; let relations = request.get("relations")?; let planning = AuditPlanning { model_locations: relations.getattr("model_locations")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index ff00ec515e..faa8aca874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -44,7 +44,7 @@ fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> /// The `CursorType` a supported cross-adapter SQL type holds, or `None`. #[pyfunction] #[pyo3(signature = (sql_type))] -fn classify_cursor_sql_type<'py>( +pub(crate) fn classify_cursor_sql_type<'py>( py: Python<'py>, sql_type: &str, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 7a9b13e5d2..562eb830b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; pub(crate) const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; @@ -14,10 +14,6 @@ pub(crate) const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; pub(crate) const COLUMN_INFO: AdapterHost = AdapterHost::ColumnInfo; pub(crate) const RELATION_LOOKUP: AdapterHost = AdapterHost::RelationLookup; pub(crate) const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; -pub(crate) const CURSOR_POLICIES: NativeAttr = NativeAttr::ResolveStartCursorConfig; -pub(crate) const MAXIMUM_START: NativeAttr = NativeAttr::MaximumAllowedStart; -pub(crate) const RENDER_SOURCE_READ: NativeAttr = NativeAttr::RenderSourceReadRelation; -pub(crate) const YAML_ENTRY_LINE: NativeAttr = NativeAttr::YamlEntryLine; pub(crate) const RUN_SCOPE: ExecutorHost = ExecutorHost::RunScope; pub(crate) const SQL_HOOK_ENTRY: RecordId = RecordId::SqlHookEntry; pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 5fff2d990a..e8e8b9cc21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,23 +1,36 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::cursor::intervals::partition; +use crate::bindings::_helpers::cursor::scalars; +use crate::bindings::_helpers::cursor::scalars::compare_values; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::cursor::scalars::to_inclusive; +use crate::bindings::_helpers::model_config::config_values::config_cursor_bound; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ - by_name, config_str, model_key, native, native_member, optional_text, planner_error, - planner_model, text, tuple, + by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, + text, tuple, }; +use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; +use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; +use crate::bindings::_helpers::planner_resolution::maximum_safety::maximum_allowed_start; +use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; use crate::bindings::_helpers::planner_warehouse::constants::{ - CURSOR_ALGEBRA_ERROR, CURSOR_POLICIES, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, - LOG_DEBUG_EVENT, MAXIMUM_START, PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, - RUN_RECORDED_QUERY, + CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, + PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, }; use crate::bindings::_helpers::planner_warehouse::snapshot::{ SnapshotRequest, inspection_concurrency, }; use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -96,19 +109,11 @@ fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &str, ) -> PyResult> { - native( - py, - NativeAttr::CursorParse, - &[("raw", raw.clone()), ("cursor_type", text(py, cursor_type))], - ) + scalars::parse(raw, &text(py, cursor_type)) } fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - native( - value.py(), - NativeAttr::CursorRender, - &[("value", value.clone())], - ) + object(value.py(), render_value(value)?) } fn grain<'py>(py: Python<'py>, value: &str) -> PyResult> { @@ -238,13 +243,12 @@ fn upstream_relation<'py>( } else if kind.eq("source")? && let Some(source) = maps.sources.get_item(&name)? { - return native( + return object( py, - NativeAttr::RenderSourceRelation, - &[ - ("entry", source.getattr("source_entry")?), - ("adapter", adapter.clone()), - ], + render_source_relation( + &source.getattr("source_entry")?, + Some(adapter).filter(|value| !value.is_none()), + )?, ); } Ok(py.None().into_bound(py)) @@ -255,15 +259,11 @@ fn lineage_reference<'py>( input_name: &Bound<'py, PyAny>, maps: &ModelMaps<'py>, ) -> PyResult> { - native( - model.py(), - NativeAttr::ResolveLineageReference, - &[ - ("model", model.clone()), - ("input_name", input_name.clone()), - ("models_by_name", maps.models.clone().into_any()), - ("functions_by_name", maps.functions.clone().into_any()), - ], + resolve_lineage_reference( + model, + &input_name.extract::()?, + maps.models.clone().into_any(), + maps.functions.clone().into_any(), ) } @@ -279,13 +279,9 @@ fn terminal_bound<'py>( let Some(upstream) = maps.models.get_item(reference.getattr("ref_name")?)? else { return Ok(None); }; - let value = native( + let value = object( py, - NativeAttr::GetConfigCursorBound, - &[ - ("values", upstream.getattr("config")?.getattr("values")?), - ("key", text(py, key)), - ], + config_cursor_bound(&upstream.getattr("config")?.getattr("values")?, key)?, )?; if value.is_none() { return Ok(None); @@ -361,12 +357,7 @@ fn collect_models<'py>( else { continue; }; - let watermark_inputs = native( - py, - NativeAttr::ResolveCursorInputRoles, - &[("model", model.clone())], - )? - .getattr("watermark_inputs")?; + let watermark_inputs = resolve_cursor_input_roles(&model)?.getattr("watermark_inputs")?; let watermark_inputs: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = watermark_inputs .call_method0("items")? .try_iter()? @@ -460,20 +451,13 @@ fn collect_models<'py>( None => py.None().into_bound(py), }); } - let effective_grain = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveTimestampGrain)?, - &[ - ("cursor_type", optional_text(py, cursor_type.as_deref())), - ( - "downstream_grain", - optional_text(py, cursor_grain.as_deref()), - ), - ( - "microbatch_strategy", - optional_text(py, strategy.as_deref()), - ), - ("cursor_input_grains", tuple(py, input_grains)?), - ], + let effective_grain = resolve_effective_timestamp_grain( + &optional_text(py, cursor_type.as_deref()), + &optional_text(py, cursor_grain.as_deref()), + &tuple(py, input_grains)?, + optional_text(py, strategy.as_deref()) + .extract::>()? + .as_deref(), )?; let mode = config_str(&model, "cursor_watermark_mode")? .filter(|value| !value.is_empty()) @@ -492,13 +476,7 @@ fn collect_models<'py>( cursor_column, upstreams, effective_grain, - start_cursor_config: call_with( - ®istry::native(py, CURSOR_POLICIES)?, - &[ - ("model", model.clone()), - ("project_config", inputs.start_cursor_config.clone()), - ], - )?, + start_cursor_config: resolve_start_cursor_config(&model, &inputs.start_cursor_config)?, has_start_override: start_override_set(&model, &inputs.cursor_overrides)?, watermark_all: watermark_mode.eq("all")?, watermark_mode, @@ -851,22 +829,23 @@ fn eligible_target_maxes<'py>( { continue; } - let allowed = call_with( - ®istry::native(py, MAXIMUM_START)?, - &[ - ("discovered_value", render(target_max)?), - ("cursor_grain", info.effective_grain.clone()), - ("invocation_time", invocation_time.clone()), - ("max_ahead", config.getattr("max_ahead")?), - ], - )?; - let horizon = parse(py, &allowed, CURSOR_TYPE_TIMESTAMP)?; - let order: i64 = native( + let allowed = text( py, - NativeAttr::CursorCompare, - &[("left", target_max.clone()), ("right", horizon.clone())], - )? - .extract()?; + &maximum_allowed_start( + py, + Some(keywords( + py, + &[ + ("discovered_value", render(target_max)?), + ("cursor_grain", info.effective_grain.clone()), + ("invocation_time", invocation_time.clone()), + ("max_ahead", config.getattr("max_ahead")?), + ], + )?), + )?, + ); + let horizon = parse(py, &allowed, CURSOR_TYPE_TIMESTAMP)?; + let order: i64 = object(py, compare_values(target_max, &horizon)?)?.extract()?; if order <= 0 { continue; } @@ -935,28 +914,19 @@ fn eligible_target_maxes<'py>( .filter(|value| !value.is_empty()) .unwrap_or(CURSOR_TYPE_TIMESTAMP); let value = parse(py, &first, cursor_type)?; - let normalized = call_with( - ®istry::native(py, NativeAttr::NormalizeCursorSnapshotGrain)?, - &[ - ( - "cursor_snapshot", - planner_model( - py, - RecordId::ModelCursorSnapshot, - &[ - ("target_max", py.None().into_bound(py)), - ("upstream_mins", PyTuple::empty(py).into_any()), - ("upstream_maxes", PyTuple::empty(py).into_any()), - ("target_eligible_max", value.clone()), - ], - )?, - ), - ( - "cursor_type", - optional_text(py, info.cursor_type.as_deref()), - ), - ("effective_grain", info.effective_grain.clone()), - ], + let normalized = normalize_cursor_snapshot_grain( + &planner_model( + py, + RecordId::ModelCursorSnapshot, + &[ + ("target_max", py.None().into_bound(py)), + ("upstream_mins", PyTuple::empty(py).into_any()), + ("upstream_maxes", PyTuple::empty(py).into_any()), + ("target_eligible_max", value.clone()), + ], + )?, + &optional_text(py, info.cursor_type.as_deref()), + &info.effective_grain.clone(), )? .getattr("target_eligible_max")?; values.insert( @@ -1030,14 +1000,7 @@ fn assemble<'py>( } } let effective_max = match &upstream.terminal_end { - Some(end) => Some(native( - py, - NativeAttr::CursorExclusiveToInclusive, - &[ - ("value", end.clone()), - ("grain", cursor_grain(py, info, upstream)?), - ], - )?), + Some(end) => Some(to_inclusive(end, &cursor_grain(py, info, upstream)?)?), None => max_value.cloned(), }; if let Some(min_value) = min_value { @@ -1090,15 +1053,7 @@ fn assemble<'py>( && let Some(max_value) = max_value { available = Some( - native( - py, - NativeAttr::CursorObservedPartition, - &[ - ("value", max_value.clone()), - ("grain", cursor_grain(py, info, upstream)?), - ], - )? - .getattr("end")?, + partition(max_value, &cursor_grain(py, info, upstream)?)?.getattr("end")?, ); } if let Some(available) = available.filter(|value| !value.is_none()) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 7fe85474b5..66de58712c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -3,7 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_phases::objects::{native, optional_text, text}; +use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; +use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{optional_text, text}; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::exceptions::{PyOSError, PyValueError}; @@ -276,20 +279,14 @@ impl MigrationFingerprintCache { let metadata_json: String = args.value("metadata_json")?.extract()?; let ref_identities = &args.value("ref_identities")?; let dialect: Option = args.value("dialect")?.extract()?; - let metadata_json: String = registry::native(py, NativeAttr::WithoutDeclaredColumnsHash)? - .call1((metadata_json,))? - .extract()?; + let metadata_json: String = without_declared_columns_hash(&metadata_json); let definition = (query_sql.clone(), metadata_json.clone()); let names = match self.ref_names.get(&definition) { Some(names) => names.clone(), None => { - let found = native( + let found = object( py, - NativeAttr::MigrationFingerprintRefNames, - &[ - ("query_sql", text(py, &query_sql)), - ("metadata_json", text(py, &metadata_json)), - ], + migration_fingerprint_ref_names(py, &query_sql, &metadata_json)?, )?; let mut names: Vec = found .try_iter()? @@ -319,9 +316,8 @@ impl MigrationFingerprintCache { for (name, identity) in &key.3 { pyo3::types::PyDictMethods::set_item(&identities, name, identity)?; } - let computed: Option = native( - py, - NativeAttr::BuildMigrationFingerprint, + let computed: Option = call_with( + ®istry::native(py, NativeAttr::BuildMigrationFingerprint)?, &[ ("query_sql", text(py, &key.0)), ("metadata_json", text(py, &key.1)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index d0a9af43c6..b6859b1eec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -3,18 +3,24 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, frozenset, model_key, native, text, tuple, }; +use crate::bindings::_helpers::planner_scope::loaders::build_upstream_intermediate_source_map; use crate::bindings::_helpers::planner_warehouse::constants::{ ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -84,19 +90,8 @@ fn relevant_state_keys<'py>( if selected.is_none() { return Ok(selected.clone()); } - let deps = registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - native( - py, - NativeAttr::TransitiveClosureMany, - &[ - ("starts", selected.clone()), - ("edges", deps), - ( - "include_starts", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], - ) + let deps = build_execution_upstream_deps(project)?.into_any(); + object(py, transitive_closure_many(selected, &deps, true, None)?) } fn resolve_database<'py>( @@ -210,13 +205,9 @@ fn metadata_name_filter( names.insert(source_table_name(&entry)?); } } - let intermediate = native( + let intermediate = object( py, - NativeAttr::BuildUpstreamIntermediateSourceMap, - &[ - ("project", project.clone()), - ("selected_keys", selected.clone()), - ], + build_upstream_intermediate_source_map(project, selected)?, )?; for entry in intermediate.call_method0("values")?.try_iter()? { names.insert(source_table_name(&entry?)?); @@ -522,9 +513,8 @@ fn gather_fingerprints<'py>( )?; for schema in schemas { let schema_lower = lower(&text(py, schema))?; - let mut set = native( + let mut set = read_latest_fingerprints(Some(&keywords( py, - NativeAttr::ReadLatestFingerprints, &[ ("connection", request.connection.clone()), ("execute", request.execute.clone()), @@ -558,7 +548,7 @@ fn gather_fingerprints<'py>( ), ("filtered_node_types", filtered_types.clone()), ], - )?; + )?))?; if read_unfiltered { let by_identity = set.getattr("fingerprints_by_identity")?; let values: Vec> = if by_identity.is_none() { @@ -571,14 +561,11 @@ fn gather_fingerprints<'py>( }; unfiltered.set_item(&schema_lower, tuple(py, values)?)?; if node_names.is_some() { - set = native( + set = select_latest_fingerprints( py, - NativeAttr::SelectLatestFingerprints, - &[ - ("fingerprint_set", set), - ("node_names", node_names_value.clone()), - ("filtered_node_types", filtered_types.clone()), - ], + &set, + node_names_value.extract::>()?, + filtered_types.extract::>()?, )?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b55cad2956..c52a74fe9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -1,16 +1,18 @@ //! Source read deferral for managed source loaders. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ - native, optional_text, planner_error, replace, text, + optional_text, planner_error, replace, text, }; use crate::bindings::_helpers::planner_warehouse::constants::{ MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, }; -use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -128,14 +130,11 @@ fn resolve_target<'py>( local_config: &Bound<'py, PyAny>, target_name: Bound<'py, PyAny>, ) -> PyResult> { - native( + resolve_target_config( project_config.py(), - NativeAttr::ResolveTargetConfig, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_name", target_name), - ], + project_config, + local_config, + &target_name, ) } @@ -192,16 +191,13 @@ fn target_field<'py>( return Ok(None); } Ok(Some( - native( + object( py, - NativeAttr::RenderProjectVarText, - &[ - ("value", effective_vars.get_item(name)?), - ( - "label", - text(py, &format!("source deferral variable '${{{name}}}'")), - ), - ], + render_project_var_text( + &effective_vars.get_item(name)?, + &text(py, &format!("source deferral variable '${{{name}}}'")) + .extract::()?, + )?, )? .extract()?, )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index ea54c1ccd9..1dc2ffc0d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -2,17 +2,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ - MISSING_SOURCE_TABLE_CODE, RENDER_SOURCE_READ, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, - YAML_ENTRY_LINE, + MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; +use pyo3::types::PyString; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; @@ -66,13 +67,7 @@ fn read_relation<'py>( adapter: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - ®istry::native(adapter.py(), RENDER_SOURCE_READ)?, - &[ - ("adapter", adapter.clone()), - ("source_entry", entry.clone()), - ], - ) + Ok(PyString::new(adapter.py(), &render_source_relation(entry, Some(adapter))?).into_any()) } fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { @@ -81,18 +76,14 @@ fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { .getattr("relative_path")? .call_method0("as_posix")? .extract()?; - let line = call_with( - ®istry::native(source.py(), YAML_ENTRY_LINE)?, - &[ - ("contents", file.getattr("contents")?), - ("name", source.getattr("name")?), - ], - )?; - if line.is_none() { - Ok(path) - } else { - Ok(format!("{path}:{}", line.str()?)) - } + let line = yaml_entry_line( + &file.getattr("contents")?.extract::()?, + &source.getattr("name")?.extract::()?, + ); + Ok(match line { + None => path, + Some(line) => format!("{path}:{line}"), + }) } /// Raise S405 when a selected resource reads an external source table that does not exist. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index f9c629d6fb..879911322b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,12 +1,12 @@ //! Validate reliable watermark column contracts before warehouse inspection. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; +use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; @@ -24,8 +24,10 @@ pub(crate) fn watermark_type_is_compatible( else { return Ok(true); }; - let classified = registry::native(declared_type.py(), NativeAttr::ClassifyCursorSqlType)? - .call1((declared_type,))?; + let classified = object( + declared_type.py(), + classify_cursor_sql_type(declared_type.py(), &declared_type.extract::()?)?, + )?; classified.eq(expected) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs index 721d165ef3..a46a8dc240 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod registration; -mod relations; +pub(crate) mod relations; pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs index 4ef70d6c7a..de66cb5576 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs @@ -70,7 +70,7 @@ fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { /// The project relation `relation` names, resolving missing qualifiers from the defaults. #[pyfunction] #[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] -fn match_project_relation<'py>( +pub(crate) fn match_project_relation<'py>( index: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, default_database: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 836e7da804..074c9f7beb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -785,7 +785,7 @@ fn degraded_document( /// Write execution JSON to stdout or a requested side-channel file. #[pyfunction] #[pyo3(signature = (*, payload, json_output, json_output_path))] -fn write_execution_json_output( +pub(crate) fn write_execution_json_output( py: Python<'_>, payload: Bound<'_, PyAny>, json_output: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs index df2a6ce8d4..78d947b28b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs @@ -287,7 +287,7 @@ impl TerminalEventClaim { /// Return the terminal event index installed for this invocation. #[pyfunction] -fn current_terminal_event_index(py: Python<'_>) -> PyResult> { +pub(crate) fn current_terminal_event_index(py: Python<'_>) -> PyResult> { context_variable(py)?.call_method0("get") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index ca768df886..f6afb5970a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -36,7 +36,7 @@ type Object<'py> = Bound<'py, PyAny>; /// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. #[pyfunction] #[pyo3(signature = (**request))] -fn source_binding_diagnostics<'py>( +pub(crate) fn source_binding_diagnostics<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("source_binding_diagnostics", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index e7bf16eaf4..35c35da692 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -48,7 +48,7 @@ type PlanRow = ( /// The Python planning request: compiled models and tests, with the adapter's facts. #[derive(FromPyObject)] -struct PlanningRequest<'py> { +pub(crate) struct PlanningRequest<'py> { #[pyo3(attribute)] models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. @@ -79,7 +79,7 @@ struct PlanningRequest<'py> { /// The Python chain request: compiled models and the tests whose chains to resolve. #[derive(FromPyObject)] -struct ChainRequest<'py> { +pub(crate) struct ChainRequest<'py> { #[pyo3(attribute)] models: Vec>, #[pyo3(attribute)] @@ -90,7 +90,7 @@ struct ChainRequest<'py> { /// Each test's ordered unmocked model chain; direct-logic tests have none. #[pyfunction] -fn resolve_compiled_sql_test_chains( +pub(crate) fn resolve_compiled_sql_test_chains( py: Python<'_>, request: ChainRequest<'_>, ) -> PyResult>> { @@ -118,7 +118,7 @@ fn resolve_compiled_sql_test_chains( /// Plan SQL tests in one batch and return each plan with its distinct error messages. #[pyfunction] -fn plan_compiled_sql_tests( +pub(crate) fn plan_compiled_sql_tests( py: Python<'_>, request: PlanningRequest<'_>, ) -> PyResult<(Vec, u64, u64)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs index bbcc55a893..22ff949e69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -25,7 +25,7 @@ pub(crate) struct SqlCodeScanner { impl SqlCodeScanner { #[new] #[pyo3(signature = (sql, syntax, context))] - fn new(sql: &str, syntax: Option, context: String) -> Self { + pub(crate) fn new(sql: &str, syntax: Option, context: String) -> Self { Self { scanner: code_scanner(sql, syntax.map(Into::into)), context, @@ -33,12 +33,12 @@ impl SqlCodeScanner { } /// The end of the comment or quoted text at `index`, or `None` when code starts there. - fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { + pub(crate) fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) } /// The index of the parenthesis closing the one at `open`. - fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { + pub(crate) fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs index 4116afb30d..8cfb088388 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -222,7 +222,9 @@ fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_histories<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_old_name_view_histories", request)?; let access: StateAccess<'py> = access(&request)?; let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -283,7 +285,7 @@ fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( +pub(crate) fn stored_columns_binding<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, database: Bound<'py, PyAny>, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index 31823edc7c..3150af5573 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -4,7 +4,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( ChangeDetectionResult, ChangeKind, @@ -69,7 +68,6 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem model_version_hashes={"orders": test_case.stale_warning_hash}, ) - monkeypatch.setattr(native_module, "detect_model_changes_in_scope", lambda **_: {}) stale_scope: PlannerScope = PlannerScope( upstream_deps={}, downstream_deps={}, @@ -81,7 +79,10 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem reused: PlannerChangeResults = detect_stale_warning_changes( project=SimpleNamespace( - settings=SimpleNamespace(query_change_tracking=test_case.query_change_tracking) + settings=SimpleNamespace(query_change_tracking=test_case.query_change_tracking), + functions=(), + hook_functions=(), + sql_analysis_dialect="duckdb", ), scopes=PlannerScopeResolution( selected_scope=stale_scope, From c54be1283cd2c37c49f579de60454887f2111d2a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:57:13 +0100 Subject: [PATCH 4/4] refactor(commands): call native helpers directly in freshness, janitor, check and diff The freshness, janitor, check, diff and deferred-target code calls the native discovery, graph, selection, connection, target, progress, state and output helpers it uses as Rust functions instead of looking them up on the native module. SQL test chain resolution is called directly; SQL test planning stays a module lookup because tests observe it there. Native classes the commands construct (progress reporters, the event writer, the cost context and freshness errors) are still created through the module. --- .../bindings/_helpers/check/check_command.rs | 16 +-- .../bindings/_helpers/check/check_results.rs | 10 +- .../_helpers/compile_driver/deferral.rs | 20 ++-- .../bindings/_helpers/compile_driver/diff.rs | 43 ++++---- .../_helpers/freshness/freshness_command.rs | 100 +++++++----------- .../_helpers/freshness/publication.rs | 26 +++-- .../_helpers/janitor/janitor_command.rs | 42 ++++---- .../_helpers/janitor/janitor_facts.rs | 10 +- .../_helpers/janitor/janitor_old_names.rs | 30 +++--- .../_helpers/sql_test_glue/plan_driver.rs | 7 +- 10 files changed, 142 insertions(+), 162 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index e3a194e6f0..b5e20e9e3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -1,5 +1,7 @@ //! `sqb check`: run the selected Python checks after their Python dependencies. +use crate::bindings::_helpers::cli::progress::current_native_progress_projector; +use crate::bindings::_helpers::cli::progress_text::write_execution_header; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; @@ -7,9 +9,11 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_python_re use crate::bindings::_helpers::compile_driver::selection_phases::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; +use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ @@ -82,12 +86,12 @@ fn empty_tuple(py: Python<'_>) -> Bound<'_, PyAny> { /// Refuse `+`, `~` and path selectors, splitting selectors as Python's `split` does. fn refuse_graph_selectors(py: Python<'_>, selectors: &[Bound<'_, PyAny>]) -> PyResult<()> { - let parse: Bound<'_, PyAny> = registry::native(py, NativeAttr::ParseProjectSelector)?; let path_selector: Bound<'_, PyAny> = registry::class(py, RecordId::PathSelector)?; for selector in selectors { for token in selector.call_method0("split")?.try_iter()? { for part in token?.call_method1("split", (",",))?.try_iter()? { - let parsed: Bound<'_, PyAny> = parse.call1((part?,))?; + let parsed: Bound<'_, PyAny> = + parse_project_selector(part?.downcast::()?)?; if parsed.is_instance(&path_selector)? || parsed.getattr("upstream")?.is_truthy()? || parsed.getattr("downstream")?.is_truthy()? @@ -197,10 +201,9 @@ fn compile_pipeline<'py>( let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let source_type: Bound<'py, PyAny> = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; - let ref_key: Bound<'py, PyAny> = registry::native(py, NativeAttr::SqlRefKey)?; let mut keys: Vec> = Vec::new(); for reference in required_refs.try_iter()? { - keys.push(ref_key.call1((reference?,))?); + keys.push(sql_ref_key(py, &reference?)?); } for source_file in discovered.getattr("source_files")?.try_iter()? { for source in source_file?.getattr("source_entries")?.try_iter()? { @@ -567,8 +570,7 @@ fn execute<'py>( let event_writer: Bound<'py, PyAny> = registry::native(py, NativeAttr::ExecutionEventWriter)?.call0()?; let executed: PyResult>> = (|| { - let projector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CurrentNativeProgressProjector)?.call0()?; + let projector: Bound<'py, PyAny> = current_native_progress_projector(py)?; if !projector.is_none() { let ordinals: Bound<'py, PyDict> = PyDict::new(py); let mut total: usize = 0; @@ -657,7 +659,7 @@ fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = prepare(request, &invocation)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs index 8688e368a1..2dbdbb94d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -14,10 +14,10 @@ use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, Check use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::json::models::JsonValue; +use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::result_output::integration_values::json_value; +use crate::bindings::_helpers::result_output::terminal_index::current_terminal_event_index; const WARN_SEVERITY: &str = "warn"; @@ -124,8 +124,7 @@ pub(crate) fn write_results( let check: Bound<'_, PyAny> = check?; checks.insert(check.getattr("name")?.extract()?, check); } - let projector: Bound<'_, PyAny> = - registry::native(py, NativeAttr::CurrentNativeProgressProjector)?.call0()?; + let projector: Bound<'_, PyAny> = current_native_progress_projector(py)?; let mut groups: Vec<(String, Vec)> = Vec::new(); for result in results { let text: CheckResultText = check_result_text(result)?; @@ -169,8 +168,7 @@ pub(crate) fn results_json(results: &[Bound<'_, PyAny>]) -> PyResult { return check_json(&[]).map_err(|error| PyValueError::new_err(format!("{error:?}"))); }; let py: Python<'_> = first.py(); - let index: Bound<'_, PyAny> = - registry::native(py, NativeAttr::CurrentTerminalEventIndex)?.call0()?; + let index: Bound<'_, PyAny> = current_terminal_event_index(py)?; let mut texts: Vec = Vec::new(); for result in results { let name: Bound<'_, PyAny> = result.getattr("node_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index b7f9b6bf1e..21dc42cdd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -9,8 +9,9 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; use sqlbuild_model_config::templates::models::DeferredSegment; +use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const PRESERVE_TARGET_VALUE: &str = "preserve"; @@ -150,24 +151,15 @@ fn target_field<'py>( .iter() .map(|key| (*key).to_owned()) .collect(); - let render: Bound<'py, PyAny> = registry::native(py, NativeAttr::RenderProjectVarText)?; let mut resolved: String = String::new(); for segment in deferred_target_segments(&template, &logical, &context_keys, &variables) { match segment { DeferredSegment::Text(text) => resolved.push_str(&text), DeferredSegment::Variable(name) => { - let text: String = call_with( - &render, - &[ - ("value", effective_vars.get_item(name.as_str())?), - ( - "label", - PyString::new(py, &format!("deferred target variable '${{{name}}}'")) - .into_any(), - ), - ], - )? - .extract()?; + let text: String = render_project_var_text( + &effective_vars.get_item(name.as_str())?, + &format!("deferred target variable '${{{name}}}'"), + )?; resolved.push_str(&text); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index a23568749f..f01496a11d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -13,11 +13,12 @@ use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_dif use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, + KeywordArguments, imported, keywords, }; use crate::bindings::_helpers::graph::views::project_lineage_views; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -203,25 +204,27 @@ fn run_diff_pipeline<'py>( &["cli_vars", "external_sql_reference_resolver"], ), )?; - let build: Bound<'py, PyAny> = registry::native(py, NativeAttr::BuildCompiledProject)?; let compile = |target: &str| -> PyResult> { - call_with( - &build, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("selected_target", arguments.value(target)?), - ( - "resolved_connection", - arguments.value("resolved_connection")?, - ), - ("no_sql_validation", arguments.value("no_sql_validation")?), - ("cli_vars", arguments.value("cli_vars")?), - ( - "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, - ), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("selected_target", arguments.value(target)?), + ( + "resolved_connection", + arguments.value("resolved_connection")?, + ), + ("no_sql_validation", arguments.value("no_sql_validation")?), + ("cli_vars", arguments.value("cli_vars")?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ], + )?), ) }; let left: Bound<'py, PyAny> = compile("from_target")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 91ceb4d2b0..15349b3765 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -15,15 +15,24 @@ use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness use sqlbuild_core::freshness_text::models::FreshnessSourceResult; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_project_graph; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; +use crate::bindings::_helpers::planner_selection::selectors::{ + expand_project_upstream_keys, select_project_graph, +}; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::result_output::documents::write_execution_json_output; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; @@ -72,17 +81,11 @@ fn source_names_for_keys( let py: Python<'_> = graph.py(); let source_type: Bound<'_, PyAny> = imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SOURCE")?; - let expand: Bound<'_, PyAny> = registry::native(py, NativeAttr::ExpandProjectUpstreamKeys)?; let upstream_deps: Bound<'_, PyAny> = graph.getattr("upstream_deps")?; for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; - let upstream: Bound<'_, PyAny> = expand.call( - (), - Some(&keywords( - py, - &[("key", &key), ("upstream_deps", &upstream_deps)], - )?), - )?; + let upstream: Bound<'_, PyAny> = + expand_project_upstream_keys(key.clone(), &upstream_deps)?.into_any(); for candidate in std::iter::once(Ok(key.clone())).chain(upstream.try_iter()?) { let candidate: Bound<'_, PyAny> = candidate?; if candidate.getattr("resource_type")?.eq(&source_type)? { @@ -99,13 +102,7 @@ fn graph_keys<'py>( ) -> PyResult> { let py: Python<'py> = graph.py(); let empty: Bound<'py, PyAny> = PyTuple::empty(py).into_any(); - registry::native(py, NativeAttr::SelectProjectGraph)?.call( - (), - Some(&keywords( - py, - &[("graph", graph), ("select", selectors), ("exclude", &empty)], - )?), - ) + Ok(select_project_graph(graph, selectors, &empty)?.into_any()) } /// Selected source names, sorted: selected or all keys and their upstream sources, minus excluded. @@ -206,19 +203,18 @@ fn observe( names: &[String], ) -> PyResult> { let py: Python<'_> = adapter.py(); - let connection_config: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ResolveProjectConnectionConfig)?.call( - (), - Some(&keywords( - py, - &[ - ("discovered_inputs", discovered), - ("project_dir", project_dir), - ("selected_target", &request.getattr("selected_target")?), - ("cli_vars", &request.getattr("cli_vars")?), - ], - )?), - )?; + let connection_config: Bound<'_, PyAny> = resolve_project_connection_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("selected_target", &request.getattr("selected_target")?), + ("cli_vars", &request.getattr("cli_vars")?), + ], + )?), + )?; let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; let observed = (|| { let project: Bound<'_, PyAny> = graph.getattr("project")?; @@ -258,11 +254,12 @@ fn write_json_output( payload: &str, (json_output, json_output_path): (bool, &Bound<'_, PyAny>), ) -> PyResult<()> { - let options: Bound<'_, PyDict> = keywords(py, &[("json_output_path", json_output_path)])?; - options.set_item("payload", format!("{payload}\n"))?; - options.set_item("json_output", json_output)?; - registry::native(py, NativeAttr::WriteExecutionJsonOutput)?.call((), Some(&options))?; - Ok(()) + write_execution_json_output( + py, + PyString::new(py, &format!("{payload}\n")).into_any(), + json_output, + json_output_path.clone(), + ) } /// Run one `sqb freshness` request. @@ -281,36 +278,19 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'_, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let discovered: Bound<'_, PyAny> = + discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; let select: Bound<'_, PyAny> = request.getattr("select")?; let exclude: Bound<'_, PyAny> = request.getattr("exclude")?; - registry::native(py, NativeAttr::RejectUnitTestSelectors)?.call( - (), - Some(&keywords( - py, - &[ - ("select", &select), - ("exclude", &exclude), - ("discovered_inputs", &discovered), - ], - )?), + reject_unit_test_selectors(&select, &exclude, &discovered)?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, )?; - let adapter_name: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ResolveEffectiveAdapterName)?.call( - (), - Some(&keywords( - py, - &[ - ("project_config", &discovered.getattr("project_config")?), - ("local_config", &discovered.getattr("local_config")?), - ], - )?), - )?; let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; - let graph: Bound<'_, PyAny> = registry::native(py, NativeAttr::BuildProjectGraph)?.call( - (), - Some(&keywords( + let graph: Bound<'_, PyAny> = build_project_graph( + py, + Some(keywords( py, &[ ("discovered_inputs", &discovered), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index 71dc47541b..2b0d374a70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -4,9 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -185,7 +187,6 @@ fn write_records<'py>( ) -> PyResult<()> { let py: Python<'py> = adapter.py(); let cost_context: Bound<'py, PyAny> = registry::native(py, NativeAttr::CostContext)?; - let write: Bound<'py, PyAny> = registry::native(py, NativeAttr::WriteSourceFreshnessRecords)?; for ((database, schema), records) in grouped { let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); for record in records { @@ -215,16 +216,19 @@ fn write_records<'py>( ], )?; within(&scope, |_| { - call_with( - &write, - &[ - ("connection", connection.clone()), - ("execute", adapter.getattr("execute")?), - ("database", database.clone()), - ("schema", schema.clone()), - ("records", PyTuple::new(py, &source_records)?.into_any()), - ("renderers", renderers(adapter)?), - ], + write_source_freshness_records( + py, + Some(keywords( + py, + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("database", database.clone()), + ("schema", schema.clone()), + ("records", PyTuple::new(py, &source_records)?.into_any()), + ("renderers", renderers(adapter)?), + ], + )?), ) })?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index d573d76e06..97cacbf43f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -21,13 +21,17 @@ use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; use sqlbuild_core::janitor_text::models::JanitorPlanText; +use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const COMMAND_NAME: &str = "janitor"; @@ -77,31 +81,25 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let discovered: Bound<'py, PyAny> = + discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; let as_target: Bound<'py, PyAny> = request.getattr("as_target")?; - let preview: Bound<'py, PyDict> = keywords( + validate_preview_target( py, - &[ - ("discovered_inputs", &discovered), - ("as_target", &as_target), - ], + discovered.clone(), + as_target.extract::>()?, + COMMAND_NAME.to_owned(), )?; - preview.set_item("command_name", COMMAND_NAME)?; - registry::native(py, NativeAttr::ValidatePreviewTarget)?.call((), Some(&preview))?; let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; - let active_target: Bound<'py, PyAny> = registry::native(py, NativeAttr::ResolveTargetName)? - .call( - (), - Some(&keywords( - py, - &[ - ("project_config", &discovered.getattr("project_config")?), - ("local_config", &discovered.getattr("local_config")?), - ("selected_target", &selected_target), - ], - )?), - )?; + let active_target: Bound<'py, PyAny> = object( + py, + resolve_target_name( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &selected_target, + )?, + )?; let use_color: bool = !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 338b9bc580..f428c2e97f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -8,6 +8,7 @@ use pyo3::types::PyBool; use sqlbuild_analysis::janitor_planning::constants::UNTRACKED_QUERY_DIFF_REASON; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, dedupe_schema_keys, folded_schema, integer, optional, record, sorted_schemas, text, tuple, @@ -16,13 +17,10 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; const RELATION_LOOKUP: AdapterHost = AdapterHost::BuildRelationLookup; const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; -const READ_LATEST_FINGERPRINTS: NativeAttr = NativeAttr::ReadLatestFingerprints; const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; @@ -295,8 +293,8 @@ fn tracked_relation_keys( &lookup, (database.as_deref(), schema, FINGERPRINT_TABLE_NAME), )?; - let read: PyResult> = call_with( - ®istry::native(py, READ_LATEST_FINGERPRINTS)?, + let read: PyResult> = read_latest_fingerprints(Some(&keywords( + py, &[ ("connection", warehouse.connection.clone()), ("execute", warehouse.adapter.getattr("execute")?), @@ -317,7 +315,7 @@ fn tracked_relation_keys( .getattr("render_read_latest_fingerprints_sql")?, ), ], - ); + )?)); let fingerprint_set: Bound<'_, PyAny> = match read { Ok(set) => set, Err(error) if error.is_instance_of::(py) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index f9944cedda..fbc0549abc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -9,19 +9,21 @@ use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_nam use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, }; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, }; +use crate::bindings::_helpers::state_rows::migration_io::{ + read_histories as read_view_histories, stored_columns_binding, +}; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; -const READ_HISTORIES: NativeAttr = NativeAttr::ReadOldNameViewHistories; -const STORED_COLUMNS: NativeAttr = NativeAttr::StoredOldNameViewColumns; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; @@ -74,17 +76,15 @@ fn read_histories<'py>( if !names.contains(OLD_NAME_VIEW_TABLE_NAME) || !names.contains(MIGRATION_TABLE_NAME) { continue; } - let stored_columns: Bound<'py, PyAny> = call_with( - ®istry::native(py, STORED_COLUMNS)?, - &[ - ("adapter", warehouse.adapter.clone()), - ("connection", warehouse.connection.clone()), - ("database", optional(py, database.as_deref())?), - ("schema", text(py, &schema)), - ], - )?; - let read: Bound<'py, PyAny> = call_with( - ®istry::native(py, READ_HISTORIES)?, + let stored_columns: Bound<'py, PyAny> = stored_columns_binding( + &warehouse.adapter, + &warehouse.connection, + optional(py, database.as_deref())?, + text(py, &schema), + )? + .into_any(); + let read = read_view_histories(Some(&keywords( + py, &[ ("connection", warehouse.connection.clone()), ("execute", warehouse.adapter.getattr("execute")?), @@ -96,7 +96,7 @@ fn read_histories<'py>( warehouse.adapter.getattr("render_qualified_name")?, ), ], - )?; + )?))?; for history in read.try_iter()? { histories.push(history?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index d10b065dd6..8a51d31475 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -4,11 +4,13 @@ use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timi use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use crate::bindings::_helpers::sql_test_glue::planning::resolve_compiled_sql_test_chains; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -33,7 +35,10 @@ pub(super) fn chains<'py>( ), ], )?; - let native = registry::native(py, NativeAttr::ResolveCompiledSqlTestChains)?.call1((request,)); + let native = request + .extract() + .and_then(|request| resolve_compiled_sql_test_chains(py, request)) + .and_then(|chains| object(py, chains)); let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => {