diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 47d9c379c9..f7b8cbfe1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -386,7 +386,7 @@ fn python_literal_sql_calls( /// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. #[pyfunction] #[pyo3(signature = (*, owner_kind, r#ref, upstream_loader_by_source = None))] -fn hard_coded_relation_remedy( +pub(crate) fn hard_coded_relation_remedy( owner_kind: &str, r#ref: &Bound<'_, PyAny>, upstream_loader_by_source: Option<&Bound<'_, PyAny>>, @@ -407,7 +407,7 @@ fn hard_coded_relation_remedy( /// The P008 message for SQL naming a relation kept only as a compatibility view. #[pyfunction] #[pyo3(signature = (*, owner_label, written, method, model_name))] -fn old_name_reference_message( +pub(crate) fn old_name_reference_message( owner_label: &str, written: &str, method: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs index 06800fcfe3..5b11d2bbb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/mod.rs @@ -1,5 +1,5 @@ mod identity; mod measurement; pub(crate) mod registration; -mod render; +pub(crate) mod render; mod schema_audits; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs index a40db46208..3e07e6b131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/render.rs @@ -30,7 +30,7 @@ fn qualified<'py>(mapping: &Bound<'py, PyAny>, name: &str) -> PyResult>) -> PyResult { +pub(crate) fn render_audit_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { let request = Request::new("render_audit_sql", request)?; let sql: String = request.get("unresolved_sql")?.extract()?; let models: Bound<'_, PyAny> = request.get("model_locations")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index 727dc3ce7e..7d3c5dc290 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -45,7 +45,7 @@ fn preview_json(value: &Bound<'_, PyAny>) -> PyResult { /// One project var value as interpolated text; structured values are rejected. #[pyfunction] #[pyo3(signature = (*, value, label))] -fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { +pub(crate) fn render_project_var_text(value: &Bound<'_, PyAny>, label: &str) -> PyResult { project_var_text(value, label) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f23c9beec0..b5e20e9e3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -1,5 +1,21 @@ //! `sqb check`: run the selected Python checks after their Python dependencies. +use crate::bindings::_helpers::cli::progress::current_native_progress_projector; +use crate::bindings::_helpers::cli::progress_text::write_execution_header; +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::relation_targets::build_project_relation_targets; +use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::compile_driver::selection_phases::compile_static_command_context; +use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; +use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; +use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; +use crate::bindings::_helpers::planner_entries::command_plans::build_relation_command_plan; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_selection::selectors::parse_project_selector; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; +use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -70,12 +86,12 @@ fn empty_tuple(py: Python<'_>) -> Bound<'_, PyAny> { /// Refuse `+`, `~` and path selectors, splitting selectors as Python's `split` does. fn refuse_graph_selectors(py: Python<'_>, selectors: &[Bound<'_, PyAny>]) -> PyResult<()> { - let parse: Bound<'_, PyAny> = registry::native(py, NativeAttr::ParseProjectSelector)?; let path_selector: Bound<'_, PyAny> = registry::class(py, RecordId::PathSelector)?; for selector in selectors { for token in selector.call_method0("split")?.try_iter()? { for part in token?.call_method1("split", (",",))?.try_iter()? { - let parsed: Bound<'_, PyAny> = parse.call1((part?,))?; + let parsed: Bound<'_, PyAny> = + parse_project_selector(part?.downcast::()?)?; if parsed.is_instance(&path_selector)? || parsed.getattr("upstream")?.is_truthy()? || parsed.getattr("downstream")?.is_truthy()? @@ -163,16 +179,14 @@ fn compile_pipeline<'py>( &graph.getattr("selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; - let closure: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::TransitiveClosureMany)?, - &[ - ("starts", check_names.clone().into_any()), - ("edges", graph.getattr("upstream_deps")?), - ( - "include_starts", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + let closure: Bound<'py, PyAny> = object( + py, + transitive_closure_many( + check_names.as_any(), + &graph.getattr("upstream_deps")?, + false, + None, + )?, )?; let loader_kind: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?.getattr("LOADER")?; @@ -187,10 +201,9 @@ fn compile_pipeline<'py>( let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let source_type: Bound<'py, PyAny> = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; - let ref_key: Bound<'py, PyAny> = registry::native(py, NativeAttr::SqlRefKey)?; let mut keys: Vec> = Vec::new(); for reference in required_refs.try_iter()? { - keys.push(ref_key.call1((reference?,))?); + keys.push(sql_ref_key(py, &reference?)?); } for source_file in discovered.getattr("source_files")?.try_iter()? { for source in source_file?.getattr("source_entries")?.try_iter()? { @@ -207,12 +220,9 @@ fn compile_pipeline<'py>( } } let keys: Bound<'py, PyAny> = PyFrozenSet::new(py, keys)?.into_any(); - let resolver: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveExternalSqlReferenceResolver)?, - &[ - ("project_dir", invocation.getattr("effective_project_dir")?), - ("discovered_inputs", discovered.clone()), - ], + let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver( + invocation.getattr("effective_project_dir")?, + discovered.clone(), )?; let options: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompilePipelineOptions)?, @@ -227,8 +237,8 @@ fn compile_pipeline<'py>( ("external_sql_reference_resolver", resolver), ], )?; - let context: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileStaticCommandContext)?, + let context: Bound<'py, PyAny> = compile_static_command_context(Some(&keywords( + py, &[ ("discovered_inputs", discovered), ("adapter", invocation.getattr("adapter")?), @@ -242,18 +252,18 @@ fn compile_pipeline<'py>( .getattr("on_progress")?, ), ], + )?))?; + let plan_output: Bound<'py, PyAny> = build_relation_command_plan( + &context.getattr("project")?, + &context.getattr("scope")?, + &context.getattr("relations")?, )?; - let plan_output: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildRelationCommandPlan)?, - &[ - ("project", context.getattr("project")?), - ("scope", context.getattr("scope")?), - ("relations", context.getattr("relations")?), - ], - )?; - call_with( - ®istry::native(py, NativeAttr::BuildStaticPipelineResult)?, - &[("context", context), ("plan_output", plan_output)], + build_static_pipeline_result( + py, + Some(keywords( + py, + &[("context", context), ("plan_output", plan_output)], + )?), ) } @@ -275,15 +285,12 @@ fn python_selection<'py>( request: &Bound<'py, PyAny>, discovered: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyFrozenSet>)> { - let py: Python<'py> = request.py(); + let _py: Python<'py> = request.py(); let python_graph: Bound<'py, PyAny> = python_node_graph(discovered)?; - call_with( - ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, - &[ - ("select", request.getattr("select")?), - ("exclude", request.getattr("exclude")?), - ("discovered_inputs", discovered.clone()), - ], + reject_unit_test_selectors( + &request.getattr("select")?, + &request.getattr("exclude")?, + discovered, )?; let check_names: Bound<'py, PyFrozenSet> = selected_check_names(request, &python_graph)?; Ok((python_graph, check_names)) @@ -314,27 +321,28 @@ fn prepare<'py>( ("python_node_names", PyFrozenSet::empty(py)?.into_any()), ], )?; - let lifecycle_plan: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonSqlRunLifecycle)?, - &[ - ("selection", selection), - ("python_graph", python_graph.clone()), - ], - )?; + let lifecycle_plan: Bound<'py, PyAny> = + build_python_sql_run_lifecycle(&selection, &python_graph)?; let relation_refs: Bound<'py, PyAny> = call_with( &python_graph.getattr("selected_sql_refs")?, &[("selected_names", check_names.clone().into_any())], )?; let project: Bound<'py, PyAny> = pipeline.getattr("project")?; let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; - let relation_targets: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonRelationTargets)?, - &[ - ("adapter", adapter.clone()), - ("project", project.clone()), - ("plan_output", plan_output.clone()), - ("required_refs", relation_refs.clone()), - ], + let relation_targets: Bound<'py, PyAny> = object( + py, + build_python_relation_targets( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("project", project.clone()), + ("plan_output", plan_output.clone()), + ("required_refs", relation_refs.clone()), + ], + )?), + )?, )?; Ok(Prepared { check_functions: PyTuple::new(py, check_functions)?.into_any(), @@ -353,9 +361,15 @@ fn prepare<'py>( ("effective_target_schema", "default_schema"), )?, ), - project_relations: call_with( - ®istry::native(py, NativeAttr::BuildProjectRelationTargets)?, - &[("adapter", adapter), ("plan_output", plan_output)], + project_relations: object( + py, + build_project_relation_targets( + py, + Some(keywords( + py, + &[("adapter", adapter), ("plan_output", plan_output)], + )?), + )?, )?, python_graph, pipeline, @@ -556,8 +570,7 @@ fn execute<'py>( let event_writer: Bound<'py, PyAny> = registry::native(py, NativeAttr::ExecutionEventWriter)?.call0()?; let executed: PyResult>> = (|| { - let projector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CurrentNativeProgressProjector)?.call0()?; + let projector: Bound<'py, PyAny> = current_native_progress_projector(py)?; if !projector.is_none() { let ordinals: Bound<'py, PyDict> = PyDict::new(py); let mut total: usize = 0; @@ -617,16 +630,14 @@ fn complete<'py>( .call_method1("__truediv__", ("target",))?, &tuple, )?; - call_with( - ®istry::native(py, NativeAttr::WriteExecutionJsonOutput)?, - &[ - ( - "payload", - PyString::new(py, &results_json(results)?).into_any(), - ), - ("json_output", request.getattr("json_output")?), - ("json_output_path", request.getattr("json_output_path")?), - ], + object( + py, + write_execution_json_output( + py, + PyString::new(py, &results_json(results)?).into_any(), + request.getattr("json_output")?.is_truthy()?, + request.getattr("json_output_path")?, + )?, )?; Ok(check_exit_code(&texts)) } @@ -648,7 +659,7 @@ fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = prepare(request, &invocation)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs index 44a352eb20..0937a8a3f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs @@ -1,5 +1,9 @@ //! Python check planning: run-state recording, dependency relations and read-side preflight. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; +use crate::bindings::_helpers::graph::python_selection::resolve_python_check_excludes; +use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::IntoPyObject; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -15,8 +19,6 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const MISSING_RELATION_CODE: &str = "C682"; const MODEL_KIND: &str = "model"; @@ -170,14 +172,20 @@ pub(crate) fn build_check_relation_targets<'py>( &python_graph.getattr("selected_sql_refs")?, &[("selected_names", selected_python_names)], )?; - call_with( - ®istry::native(py, NativeAttr::BuildPythonRelationTargets)?, - &[ - ("adapter", adapter.clone()), - ("project", pipeline_result.getattr("project")?), - ("plan_output", pipeline_result.getattr("plan_output")?), - ("required_refs", required), - ], + object( + py, + build_python_relation_targets( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("project", pipeline_result.getattr("project")?), + ("plan_output", pipeline_result.getattr("plan_output")?), + ("required_refs", required), + ], + )?), + )?, ) } @@ -189,9 +197,9 @@ pub(crate) fn relevant_check_functions<'py>( (selected_dependency_names, completed_sql_names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult>> { let py: Python<'py> = discovered_inputs.py(); - let excluded: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolvePythonCheckExcludes)?, - &[("graph", python_graph.clone()), ("exclude", exclude)], + let excluded: Bound<'py, PyAny> = object( + py, + resolve_python_check_excludes(python_graph, exclude.extract::>()?)?, )?; let check_kind: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?.getattr("CHECK")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs index 8688e368a1..2dbdbb94d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs @@ -14,10 +14,10 @@ use sqlbuild_core::check_command::models::{CheckGrouping, CheckResultText, Check use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::json::models::JsonValue; +use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::result_output::integration_values::json_value; +use crate::bindings::_helpers::result_output::terminal_index::current_terminal_event_index; const WARN_SEVERITY: &str = "warn"; @@ -124,8 +124,7 @@ pub(crate) fn write_results( let check: Bound<'_, PyAny> = check?; checks.insert(check.getattr("name")?.extract()?, check); } - let projector: Bound<'_, PyAny> = - registry::native(py, NativeAttr::CurrentNativeProgressProjector)?.call0()?; + let projector: Bound<'_, PyAny> = current_native_progress_projector(py)?; let mut groups: Vec<(String, Vec)> = Vec::new(); for result in results { let text: CheckResultText = check_result_text(result)?; @@ -169,8 +168,7 @@ pub(crate) fn results_json(results: &[Bound<'_, PyAny>]) -> PyResult { return check_json(&[]).map_err(|error| PyValueError::new_err(format!("{error:?}"))); }; let py: Python<'_> = first.py(); - let index: Bound<'_, PyAny> = - registry::native(py, NativeAttr::CurrentTerminalEventIndex)?.call0()?; + let index: Bound<'_, PyAny> = current_terminal_event_index(py)?; let mut texts: Vec = Vec::new(); for result in results { let name: Bound<'_, PyAny> = result.getattr("node_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index bf1bb54efb..8236109929 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -314,7 +314,7 @@ fn context_variable(py: Python<'_>) -> PyResult> { /// Return the projector installed for the current CLI invocation. #[pyfunction] -fn current_native_progress_projector(py: Python<'_>) -> PyResult> { +pub(crate) fn current_native_progress_projector(py: Python<'_>) -> PyResult> { context_variable(py)?.call_method0("get") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index a5e428ab38..2792dbbead 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -35,7 +35,10 @@ fn format_execution_header(command: &str, target: Option<&str>, concurrency: i64 /// Write an execution header for command progress output. #[pyfunction] #[pyo3(signature = (**options))] -fn write_execution_header(py: Python<'_>, options: Option>) -> PyResult<()> { +pub(crate) fn write_execution_header( + py: Python<'_>, + options: Option>, +) -> PyResult<()> { let arguments: KeywordArguments<'_> = KeywordArguments::read(py, "write_execution_header", options, HEADER_OPTIONS)?; let command: String = arguments.value("command")?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 2ccae8393e..d1ae0ad2a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -279,7 +279,7 @@ pub(crate) fn project_connection_config<'py>( /// Resolve the project connection; offline modes pass `require_connection=False`. #[pyfunction] #[pyo3(signature = (**options))] -fn resolve_project_connection_config<'py>( +pub(crate) fn resolve_project_connection_config<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs index 0bfa4096cf..a57cd9990e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/preview_target.rs @@ -19,7 +19,7 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { /// Fail clearly when `--as` names a target that is not configured. #[pyfunction] #[pyo3(signature = (*, discovered_inputs, as_target, command_name))] -fn validate_preview_target( +pub(crate) fn validate_preview_target( py: Python<'_>, discovered_inputs: Bound<'_, PyAny>, as_target: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 3f5f2f5bd3..21dc42cdd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -9,8 +9,9 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments; use sqlbuild_model_config::templates::models::DeferredSegment; +use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const PRESERVE_TARGET_VALUE: &str = "preserve"; @@ -29,7 +30,7 @@ struct DeferredNaming<'py> { /// Build physical locations for all models and seeds under a deferred target. #[pyfunction] #[pyo3(signature = (**options))] -fn build_deferred_locations<'py>( +pub(crate) fn build_deferred_locations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -150,24 +151,15 @@ fn target_field<'py>( .iter() .map(|key| (*key).to_owned()) .collect(); - let render: Bound<'py, PyAny> = registry::native(py, NativeAttr::RenderProjectVarText)?; let mut resolved: String = String::new(); for segment in deferred_target_segments(&template, &logical, &context_keys, &variables) { match segment { DeferredSegment::Text(text) => resolved.push_str(&text), DeferredSegment::Variable(name) => { - let text: String = call_with( - &render, - &[ - ("value", effective_vars.get_item(name.as_str())?), - ( - "label", - PyString::new(py, &format!("deferred target variable '${{{name}}}'")) - .into_any(), - ), - ], - )? - .extract()?; + let text: String = render_project_var_text( + &effective_vars.get_item(name.as_str())?, + &format!("deferred target variable '${{{name}}}'"), + )?; resolved.push_str(&text); } } @@ -186,7 +178,7 @@ fn planner_error(py: Python<'_>, message: String) -> PyErr { /// Validate and resolve the deferred target config. #[pyfunction] #[pyo3(signature = (**options))] -fn resolve_deferred_target_config<'py>( +pub(crate) fn resolve_deferred_target_config<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -233,7 +225,7 @@ fn deferred_target_config<'py>( /// Gather existing relations from the deferred target's relation-location schemas. #[pyfunction] #[pyo3(signature = (**options))] -fn gather_deferred_relations<'py>( +pub(crate) fn gather_deferred_relations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index a23568749f..f01496a11d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -13,11 +13,12 @@ use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_dif use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, + KeywordArguments, imported, keywords, }; use crate::bindings::_helpers::graph::views::project_lineage_views; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -203,25 +204,27 @@ fn run_diff_pipeline<'py>( &["cli_vars", "external_sql_reference_resolver"], ), )?; - let build: Bound<'py, PyAny> = registry::native(py, NativeAttr::BuildCompiledProject)?; let compile = |target: &str| -> PyResult> { - call_with( - &build, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("selected_target", arguments.value(target)?), - ( - "resolved_connection", - arguments.value("resolved_connection")?, - ), - ("no_sql_validation", arguments.value("no_sql_validation")?), - ("cli_vars", arguments.value("cli_vars")?), - ( - "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, - ), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("selected_target", arguments.value(target)?), + ( + "resolved_connection", + arguments.value("resolved_connection")?, + ), + ("no_sql_validation", arguments.value("no_sql_validation")?), + ("cli_vars", arguments.value("cli_vars")?), + ( + "external_sql_reference_resolver", + arguments.value("external_sql_reference_resolver")?, + ), + ], + )?), ) }; let left: Bound<'py, PyAny> = compile("from_target")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index d422e28f3c..256315d1ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -165,7 +165,7 @@ struct DiscoveryRequest { /// Load all raw project inputs from disk before semantic resolution. #[pyfunction] #[pyo3(signature = (**options))] -fn discover_project_inputs<'py>( +pub(crate) fn discover_project_inputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 9ffdda1e23..64231949f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -84,7 +84,7 @@ pub(crate) fn compile_analysis_selection<'py>( /// Build a static dependency graph for an already-compiled project. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph_from_compiled_project<'py>( +pub(crate) fn build_project_graph_from_compiled_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -136,7 +136,7 @@ fn plan_has_executable_work<'py>( /// Wrap a focused command projection in the established pipeline envelope. #[pyfunction] #[pyo3(signature = (**options))] -fn build_static_pipeline_result<'py>( +pub(crate) fn build_static_pipeline_result<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 142b6a0df1..89409309b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -1,5 +1,12 @@ //! Plan a compiled project over an open warehouse connection, and open that connection. +use crate::bindings::_helpers::compile_driver::deferral::build_deferred_locations; +use crate::bindings::_helpers::compile_driver::deferral::gather_deferred_relations; +use crate::bindings::_helpers::compile_driver::deferral::resolve_deferred_target_config; +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::python_plan_entries::python_read_source_names_for; +use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; @@ -50,20 +57,23 @@ pub(crate) fn pipeline_result<'py>( .getattr("resolve_python_run_selectors")? .is_truthy()? { - let graph: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildProjectGraphFromCompiledProject)?, - &[("project", project.clone())], + let graph: Bound<'py, PyAny> = build_project_graph_from_compiled_project( + py, + Some(keywords(py, &[("project", project.clone())])?), )?; run_selection = run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; selected_sql_keys = run_selection.getattr("sql_keys")?; selected_python_node_names = run_selection.getattr("python_node_names")?; - python_read_source_names = call_with( - ®istry::native(py, NativeAttr::PythonReadSourceNamesFor)?, - &[ - ("discovered_inputs", discovered.clone()), - ("python_node_names", selected_python_node_names.clone()), - ], + python_read_source_names = python_read_source_names_for( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("python_node_names", selected_python_node_names.clone()), + ], + )?), )?; } let selection: Bound<'py, PyAny> = call_with( @@ -179,41 +189,56 @@ fn deferral<'py>( return Ok((py.None().into_bound(py), py.None().into_bound(py))); } let adapter: &Bound<'py, PyAny> = inputs.adapter; - let target_config: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveDeferredTargetConfig)?, - &[ - ("discovered_inputs", inputs.discovered_inputs.clone()), - ("defer_to", defer_to.clone()), - ( - "current_target_name", - project.getattr("effective_target_name")?, - ), - ], + let target_config: Bound<'py, PyAny> = resolve_deferred_target_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", inputs.discovered_inputs.clone()), + ("defer_to", defer_to.clone()), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?), )?; - let locations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildDeferredLocations)?, - &[ - ("project", project.clone()), - ("deferred_target_config", target_config), - ("effective_vars", project.getattr("effective_vars")?), - ("default_schema", adapter.call_method0("default_schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ], + let locations: Bound<'py, PyAny> = object( + py, + build_deferred_locations( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("deferred_target_config", target_config), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + )?), + )?, )?; - let relations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::GatherDeferredRelations)?, - &[ - ("adapter", adapter.clone()), - ("connection", inputs.connection.clone()), - ("deferred_locations", locations.clone()), - ], + let relations: Bound<'py, PyAny> = object( + py, + gather_deferred_relations( + py, + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("connection", inputs.connection.clone()), + ("deferred_locations", locations.clone()), + ], + )?), + )?, )?; Ok((locations, relations)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 276d018fa5..ca9031ae7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -71,7 +71,7 @@ fn py_bool(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { /// Build one compiled project with adapter defaults and target validation applied. #[pyfunction] #[pyo3(signature = (**options))] -fn build_compiled_project<'py>( +pub(crate) fn build_compiled_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -318,7 +318,7 @@ fn build_project_graph_with_analysis_selection<'py>( /// Build the static dependency graph for a compiled project. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph<'py>( +pub(crate) fn build_project_graph<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 387db3dbef..625a1a4cac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -1,5 +1,6 @@ //! Display-ready Python plan entries and Python-aware selection warnings of a plan. +use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; use std::collections::HashMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -15,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const SKIPPED_UPSTREAM_CODE: &str = "P501"; @@ -68,7 +69,7 @@ fn node_graph<'py>(discovered: &Bound<'py, PyAny>) -> PyResult /// Return sources declared by the selected Python nodes and their Python upstreams. #[pyfunction] #[pyo3(signature = (**options))] -fn python_read_source_names_for<'py>( +pub(crate) fn python_read_source_names_for<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -104,7 +105,7 @@ fn python_read_source_names_for<'py>( /// Attach python-aware selection warnings and plan entries to the plan output. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_run_plan_outputs<'py>( +pub(crate) fn build_python_run_plan_outputs<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -161,10 +162,7 @@ fn build_python_run_plan_outputs<'py>( ("python_node_names", selected.clone()), ], )?; - let lifecycle: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonSqlRunLifecycle)?, - &[("selection", selection), ("python_graph", graph.clone())], - )?; + let lifecycle: Bound<'py, PyAny> = build_python_sql_run_lifecycle(&selection, &graph)?; let entries: Bound<'py, PyTuple> = plan_entries( &lifecycle, (&graph, &kinds), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index ff9880d2d4..03045bde39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -47,7 +47,7 @@ fn shared_error(py: Python<'_>, message: String) -> PyErr { /// Return adapter-qualified runtime relations required by selected Python nodes. #[pyfunction] #[pyo3(signature = (**options))] -fn build_python_relation_targets<'py>( +pub(crate) fn build_python_relation_targets<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -80,7 +80,7 @@ fn build_python_relation_targets<'py>( /// Return every planned project relation for hard-coded name warnings, or None when off. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_relation_targets<'py>( +pub(crate) fn build_project_relation_targets<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 65b0c4e43c..7776309998 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -211,7 +211,7 @@ fn deferred_locations<'py>( /// Run canonical compile, selector, and relation phases without build planning. #[pyfunction] #[pyo3(signature = (**request))] -fn compile_static_command_context<'py>( +pub(crate) fn compile_static_command_context<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request: Request<'_, 'py> = Request::new("compile_static_command_context", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index dd1c27fc9f..731a22f154 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -309,7 +309,7 @@ fn family_accepts(family: &Bound<'_, PyAny>, column_name: &str) -> PyResult( +pub(crate) fn matching_dynamic_families<'py>( families: &Bound<'py, PyAny>, column_name: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 2a940a21e7..fadaa324d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -63,7 +63,7 @@ pub(crate) fn interval_validation( /// The canonical partition containing an observed physical value. #[pyfunction(name = "cursor_observed_partition")] #[pyo3(signature = (*, value, grain))] -fn partition<'py>( +pub(crate) fn partition<'py>( value: &Bound<'py, PyAny>, grain: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index 3648587691..8225c1b08b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -187,7 +187,7 @@ pub(crate) fn as_scalar<'py>( /// One cursor scalar, preserving date-ness and timestamp offsets. #[pyfunction(name = "cursor_parse")] #[pyo3(signature = (*, raw, cursor_type))] -fn parse<'py>( +pub(crate) fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, ) -> PyResult> { @@ -248,7 +248,7 @@ fn sentinel_from_token<'py>(token: &Bound<'py, PyAny>) -> PyResult, right: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { let session: CursorSession<'_> = CursorSession::new(left.py())?; let (left, right): (Scalar, Scalar) = (session.scalar(left)?, session.scalar(right)?); Ok(match session.checked(compare(&left, &right, &session))? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 4fadf5da43..0cd7da5cd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -48,7 +48,7 @@ fn sql_analysis_off_guidance( } #[pyfunction] -fn microbatch_guidance() -> (String, String) { +pub(crate) fn microbatch_guidance() -> (String, String) { concurrency_guidance() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index 976d68d0ef..f0fd7178d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -8,7 +8,7 @@ use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; /// The 1-based line of the outermost `- name: ` entry, if present. #[pyfunction] #[pyo3(signature = (*, contents, name))] -fn yaml_entry_line(contents: &str, name: &str) -> Option { +pub(crate) fn yaml_entry_line(contents: &str, name: &str) -> Option { yaml_entry_lines(contents) .into_iter() .find_map(|(entry, line)| (entry == name).then_some(line)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index cc20c8e1c3..398a2e3074 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -56,7 +56,7 @@ pub(crate) fn fingerprint_set_from_rows<'py>( /// `select_latest_fingerprints`: an unfiltered latest read narrowed as a filtered read would be. #[pyfunction] #[pyo3(signature = (*, fingerprint_set, node_names, filtered_node_types = Vec::new()))] -fn select_latest_fingerprints<'py>( +pub(crate) fn select_latest_fingerprints<'py>( py: Python<'py>, fingerprint_set: &Bound<'py, PyAny>, node_names: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs index dbf133c673..846cc60777 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod latest_rows; -mod query_hashes; +pub(crate) mod query_hashes; pub(crate) mod registration; -mod storage_io; +pub(crate) mod storage_io; pub(crate) mod storage_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs index 9532cbf23b..743cc60100 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/query_hashes.rs @@ -27,7 +27,11 @@ fn query_fingerprint_error(py: Python<'_>, message: String) -> PyErr { /// The persisted SHA-256 fingerprint of the query's canonical token stream. #[pyfunction] #[pyo3(signature = (*, query_sql, dialect))] -fn compute_query_hash(py: Python<'_>, query_sql: &str, dialect: Option<&str>) -> PyResult { +pub(crate) fn compute_query_hash( + py: Python<'_>, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) .map_err(|error| query_fingerprint_error(py, error)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 01ea896109..10890995c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -291,7 +291,7 @@ fn unreadable_table(py: Python<'_>, qualified_name: &str, error: PyErr) -> PyErr /// The latest fingerprint per node identity, read with the adapter-rendered latest-row SQL. #[pyfunction] #[pyo3(signature = (**request))] -fn read_latest_fingerprints<'py>( +pub(crate) fn read_latest_fingerprints<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("read_latest_fingerprints", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 91ceb4d2b0..15349b3765 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -15,15 +15,24 @@ use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness use sqlbuild_core::freshness_text::models::FreshnessSourceResult; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_project_graph; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; use crate::bindings::_helpers::freshness::observation::Observer; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; +use crate::bindings::_helpers::planner_selection::selectors::{ + expand_project_upstream_keys, select_project_graph, +}; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::result_output::documents::write_execution_json_output; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; @@ -72,17 +81,11 @@ fn source_names_for_keys( let py: Python<'_> = graph.py(); let source_type: Bound<'_, PyAny> = imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SOURCE")?; - let expand: Bound<'_, PyAny> = registry::native(py, NativeAttr::ExpandProjectUpstreamKeys)?; let upstream_deps: Bound<'_, PyAny> = graph.getattr("upstream_deps")?; for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; - let upstream: Bound<'_, PyAny> = expand.call( - (), - Some(&keywords( - py, - &[("key", &key), ("upstream_deps", &upstream_deps)], - )?), - )?; + let upstream: Bound<'_, PyAny> = + expand_project_upstream_keys(key.clone(), &upstream_deps)?.into_any(); for candidate in std::iter::once(Ok(key.clone())).chain(upstream.try_iter()?) { let candidate: Bound<'_, PyAny> = candidate?; if candidate.getattr("resource_type")?.eq(&source_type)? { @@ -99,13 +102,7 @@ fn graph_keys<'py>( ) -> PyResult> { let py: Python<'py> = graph.py(); let empty: Bound<'py, PyAny> = PyTuple::empty(py).into_any(); - registry::native(py, NativeAttr::SelectProjectGraph)?.call( - (), - Some(&keywords( - py, - &[("graph", graph), ("select", selectors), ("exclude", &empty)], - )?), - ) + Ok(select_project_graph(graph, selectors, &empty)?.into_any()) } /// Selected source names, sorted: selected or all keys and their upstream sources, minus excluded. @@ -206,19 +203,18 @@ fn observe( names: &[String], ) -> PyResult> { let py: Python<'_> = adapter.py(); - let connection_config: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ResolveProjectConnectionConfig)?.call( - (), - Some(&keywords( - py, - &[ - ("discovered_inputs", discovered), - ("project_dir", project_dir), - ("selected_target", &request.getattr("selected_target")?), - ("cli_vars", &request.getattr("cli_vars")?), - ], - )?), - )?; + let connection_config: Bound<'_, PyAny> = resolve_project_connection_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir), + ("selected_target", &request.getattr("selected_target")?), + ("cli_vars", &request.getattr("cli_vars")?), + ], + )?), + )?; let connection: Bound<'_, PyAny> = adapter.call_method1("connect", (connection_config,))?; let observed = (|| { let project: Bound<'_, PyAny> = graph.getattr("project")?; @@ -258,11 +254,12 @@ fn write_json_output( payload: &str, (json_output, json_output_path): (bool, &Bound<'_, PyAny>), ) -> PyResult<()> { - let options: Bound<'_, PyDict> = keywords(py, &[("json_output_path", json_output_path)])?; - options.set_item("payload", format!("{payload}\n"))?; - options.set_item("json_output", json_output)?; - registry::native(py, NativeAttr::WriteExecutionJsonOutput)?.call((), Some(&options))?; - Ok(()) + write_execution_json_output( + py, + PyString::new(py, &format!("{payload}\n")).into_any(), + json_output, + json_output_path.clone(), + ) } /// Run one `sqb freshness` request. @@ -281,36 +278,19 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'_, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let discovered: Bound<'_, PyAny> = + discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; let select: Bound<'_, PyAny> = request.getattr("select")?; let exclude: Bound<'_, PyAny> = request.getattr("exclude")?; - registry::native(py, NativeAttr::RejectUnitTestSelectors)?.call( - (), - Some(&keywords( - py, - &[ - ("select", &select), - ("exclude", &exclude), - ("discovered_inputs", &discovered), - ], - )?), + reject_unit_test_selectors(&select, &exclude, &discovered)?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, )?; - let adapter_name: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ResolveEffectiveAdapterName)?.call( - (), - Some(&keywords( - py, - &[ - ("project_config", &discovered.getattr("project_config")?), - ("local_config", &discovered.getattr("local_config")?), - ], - )?), - )?; let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; - let graph: Bound<'_, PyAny> = registry::native(py, NativeAttr::BuildProjectGraph)?.call( - (), - Some(&keywords( + let graph: Bound<'_, PyAny> = build_project_graph( + py, + Some(keywords( py, &[ ("discovered_inputs", &discovered), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index 71dc47541b..2b0d374a70 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -4,9 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -185,7 +187,6 @@ fn write_records<'py>( ) -> PyResult<()> { let py: Python<'py> = adapter.py(); let cost_context: Bound<'py, PyAny> = registry::native(py, NativeAttr::CostContext)?; - let write: Bound<'py, PyAny> = registry::native(py, NativeAttr::WriteSourceFreshnessRecords)?; for ((database, schema), records) in grouped { let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); for record in records { @@ -215,16 +216,19 @@ fn write_records<'py>( ], )?; within(&scope, |_| { - call_with( - &write, - &[ - ("connection", connection.clone()), - ("execute", adapter.getattr("execute")?), - ("database", database.clone()), - ("schema", schema.clone()), - ("records", PyTuple::new(py, &source_records)?.into_any()), - ("renderers", renderers(adapter)?), - ], + write_source_freshness_records( + py, + Some(keywords( + py, + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("database", database.clone()), + ("schema", schema.clone()), + ("records", PyTuple::new(py, &source_records)?.into_any()), + ("renderers", renderers(adapter)?), + ], + )?), ) })?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 1d6b3465e9..854795f961 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -261,7 +261,7 @@ fn execute_sql( /// Append source freshness rows, creating the table once if needed. #[pyfunction] #[pyo3(signature = (**options))] -fn write_source_freshness_records<'py>( +pub(crate) fn write_source_freshness_records<'py>( py: Python<'py>, options: Option>, ) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index 28dd9b0966..2b1fda2282 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -50,7 +50,7 @@ fn neighbours<'py>( /// Keys within `max_depth` edges of any start, plus the starts when requested. #[pyfunction] #[pyo3(signature = (*, starts, edges, include_starts, max_depth = None))] -fn transitive_closure_many<'py>( +pub(crate) fn transitive_closure_many<'py>( starts: &Bound<'py, PyAny>, edges: &Bound<'py, PyAny>, include_starts: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index d81e6923b7..603a7870aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -118,7 +118,10 @@ fn model_python_hook_names<'py>( /// The compiled graph key a typed model, source or seed reference names. #[pyfunction] -fn sql_ref_key<'py>(py: Python<'py>, reference: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn sql_ref_key<'py>( + py: Python<'py>, + reference: &Bound<'py, PyAny>, +) -> PyResult> { let resource_type: Bound<'py, PyAny> = py .import(COMPILE_TYPES_MODULE)? .getattr("CompiledResourceType")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs index 40cb605076..e0d28601d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -1,9 +1,9 @@ pub(crate) mod algorithms; -mod execution_gates; +pub(crate) mod execution_gates; pub(crate) mod keys; pub(crate) mod node_set; pub(crate) mod project_graph; -mod python_lifecycle; +pub(crate) mod python_lifecycle; pub(crate) mod python_node_graph; pub(crate) mod python_selection; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs index dc4c16371c..d08a2c659b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -15,7 +15,7 @@ const EXTERNAL_CONNECTION_MODE: &str = "external"; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. #[pyfunction] #[pyo3(signature = (*, selection, python_graph))] -fn build_python_sql_run_lifecycle<'py>( +pub(crate) fn build_python_sql_run_lifecycle<'py>( selection: &Bound<'py, PyAny>, python_graph: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index b8b1d112d7..851997d86b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -143,7 +143,7 @@ fn resolve_python_node_selection<'py>( /// The Python node names `sqb check` exclude tokens name, skipping tokens that match nothing. #[pyfunction] #[pyo3(signature = (*, graph, exclude))] -fn resolve_python_check_excludes<'py>( +pub(crate) fn resolve_python_check_excludes<'py>( graph: &Bound<'py, PyAny>, exclude: Vec, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index d573d76e06..97cacbf43f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -21,13 +21,17 @@ use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; use sqlbuild_core::janitor_text::models::JanitorPlanText; +use crate::bindings::_helpers::command_runtime::preview_target::validate_preview_target; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::janitor::janitor_phases::{ JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const COMMAND_NAME: &str = "janitor"; @@ -77,31 +81,25 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&keywords(py, &[("project_dir", &project_dir)])?))?; + let discovered: Bound<'py, PyAny> = + discover_project_inputs(py, Some(keywords(py, &[("project_dir", &project_dir)])?))?; let as_target: Bound<'py, PyAny> = request.getattr("as_target")?; - let preview: Bound<'py, PyDict> = keywords( + validate_preview_target( py, - &[ - ("discovered_inputs", &discovered), - ("as_target", &as_target), - ], + discovered.clone(), + as_target.extract::>()?, + COMMAND_NAME.to_owned(), )?; - preview.set_item("command_name", COMMAND_NAME)?; - registry::native(py, NativeAttr::ValidatePreviewTarget)?.call((), Some(&preview))?; let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; - let active_target: Bound<'py, PyAny> = registry::native(py, NativeAttr::ResolveTargetName)? - .call( - (), - Some(&keywords( - py, - &[ - ("project_config", &discovered.getattr("project_config")?), - ("local_config", &discovered.getattr("local_config")?), - ("selected_target", &selected_target), - ], - )?), - )?; + let active_target: Bound<'py, PyAny> = object( + py, + resolve_target_name( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &selected_target, + )?, + )?; let use_color: bool = !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index 338b9bc580..f428c2e97f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -8,6 +8,7 @@ use pyo3::types::PyBool; use sqlbuild_analysis::janitor_planning::constants::UNTRACKED_QUERY_DIFF_REASON; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, dedupe_schema_keys, folded_schema, integer, optional, record, sorted_schemas, text, tuple, @@ -16,13 +17,10 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ const QUERY_ARTIFACT_LIFECYCLE: ExecutorHost = ExecutorHost::QueryDiffArtifactLifecycle; const RELATION_LOOKUP: AdapterHost = AdapterHost::BuildRelationLookup; const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; -const READ_LATEST_FINGERPRINTS: NativeAttr = NativeAttr::ReadLatestFingerprints; const EXECUTION_LOGGER: &str = "sqlbuild.execution"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; @@ -295,8 +293,8 @@ fn tracked_relation_keys( &lookup, (database.as_deref(), schema, FINGERPRINT_TABLE_NAME), )?; - let read: PyResult> = call_with( - ®istry::native(py, READ_LATEST_FINGERPRINTS)?, + let read: PyResult> = read_latest_fingerprints(Some(&keywords( + py, &[ ("connection", warehouse.connection.clone()), ("execute", warehouse.adapter.getattr("execute")?), @@ -317,7 +315,7 @@ fn tracked_relation_keys( .getattr("render_read_latest_fingerprints_sql")?, ), ], - ); + )?)); let fingerprint_set: Bound<'_, PyAny> = match read { Ok(set) => set, Err(error) if error.is_instance_of::(py) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index f9944cedda..fbc0549abc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -9,19 +9,21 @@ use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_nam use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, }; use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, }; +use crate::bindings::_helpers::state_rows::migration_io::{ + read_histories as read_view_histories, stored_columns_binding, +}; const IS_COMPATIBILITY_VIEW: ExecutorHost = ExecutorHost::IsCompatibilityView; -const READ_HISTORIES: NativeAttr = NativeAttr::ReadOldNameViewHistories; -const STORED_COLUMNS: NativeAttr = NativeAttr::StoredOldNameViewColumns; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use sqlbuild_core::constants::MIGRATION_TABLE_NAME; use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; @@ -74,17 +76,15 @@ fn read_histories<'py>( if !names.contains(OLD_NAME_VIEW_TABLE_NAME) || !names.contains(MIGRATION_TABLE_NAME) { continue; } - let stored_columns: Bound<'py, PyAny> = call_with( - ®istry::native(py, STORED_COLUMNS)?, - &[ - ("adapter", warehouse.adapter.clone()), - ("connection", warehouse.connection.clone()), - ("database", optional(py, database.as_deref())?), - ("schema", text(py, &schema)), - ], - )?; - let read: Bound<'py, PyAny> = call_with( - ®istry::native(py, READ_HISTORIES)?, + let stored_columns: Bound<'py, PyAny> = stored_columns_binding( + &warehouse.adapter, + &warehouse.connection, + optional(py, database.as_deref())?, + text(py, &schema), + )? + .into_any(); + let read = read_view_histories(Some(&keywords( + py, &[ ("connection", warehouse.connection.clone()), ("execute", warehouse.adapter.getattr("execute")?), @@ -96,7 +96,7 @@ fn read_histories<'py>( warehouse.adapter.getattr("render_qualified_name")?, ), ], - )?; + )?))?; for history in read.try_iter()? { histories.push(history?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 1838d1a276..9aaeeeac51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -1,5 +1,7 @@ //! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -87,17 +89,20 @@ pub(crate) fn compile_janitor_project<'py>( let project: Bound<'py, PyAny> = within(&lifecycle(py, ("project", "project_compile"))?, |_| { status.call_method1("on_progress", ("Compiling project...",))?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ( - "discovered_inputs", - invocation.getattr("discovered_inputs")?, - ), - ("adapter", context.getattr("adapter")?), - ("selected_target", target), - ("resolved_connection", context.getattr("connection_config")?), - ], + let project: Bound<'py, PyAny> = build_compiled_project( + py, + Some(keywords( + py, + &[ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("adapter", context.getattr("adapter")?), + ("selected_target", target), + ("resolved_connection", context.getattr("connection_config")?), + ], + )?), )?; status.call_method1( "on_progress", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index 0d964e8bab..389dbc19b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -91,7 +91,7 @@ pub(crate) fn inclusive_cursor_end( /// A fully bounded inclusive operator override as an executable interval. #[pyfunction] #[pyo3(signature = (*, start_cursor_override, end_cursor_override, cursor_type, cursor_grain))] -fn resolve_bounded_cursor_override<'py>( +pub(crate) fn resolve_bounded_cursor_override<'py>( start_cursor_override: &Bound<'py, PyAny>, end_cursor_override: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 09b0ad43d3..ac8e45266c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,6 +1,7 @@ //! Detect how each selected model and function changed since its recorded build. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::contracts::spec_resolution::matching_dynamic_families; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -42,7 +43,7 @@ use crate::bindings::_helpers::planner_identity::metadata::{ compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, model_version_identity_metadata_json, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; @@ -156,7 +157,7 @@ pub(crate) fn detect_changes<'py>( /// Selected model changes, without inspecting functions. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_model_changes_in_scope<'py>( +pub(crate) fn detect_model_changes_in_scope<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -190,7 +191,7 @@ fn detect_model_changes_in_scope<'py>( /// One model's change and backfill. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_model_changes<'py>( +pub(crate) fn detect_model_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -594,23 +595,15 @@ fn schema_findings_of<'py>( (type_enforcement, inferred_schema_complete): (bool, bool), (dynamic, dialect): (Option>, &Bound<'py, PyAny>), ) -> PyResult> { - let py: Python<'py> = dialect.py(); + let _py: Python<'py> = dialect.py(); let dialect_name: Option = dialect.extract()?; let mut types_equal = |left: &str, right: &str| -> PyResult { native_types_equal(dialect, left, right) }; - let matching_function: Bound<'py, PyAny> = - registry::native(py, NativeAttr::MatchingDynamicFamilies)?; let mut single_family_type = |name: &str| -> PyResult> { let Some(families) = &dynamic else { return Ok(None); }; - let matching: Bound<'py, PyAny> = call_with( - &matching_function, - &[ - ("families", families.clone()), - ("column_name", PyString::new(py, name).into_any()), - ], - )?; + let matching = matching_dynamic_families(families, name)?; if matching.len()? != 1 { return Ok(None); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs index e760bb22da..2320bd9193 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/metadata.rs @@ -35,7 +35,7 @@ fn with_declared_columns_hash(metadata_json: &str, declared_columns_hash: &str) /// Fingerprint metadata JSON without the declared-columns hash key, if present. #[pyfunction] -fn without_declared_columns_hash(metadata_json: &str) -> String { +pub(crate) fn without_declared_columns_hash(metadata_json: &str) -> String { without_hash(metadata_json) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 47462cf513..12d79a6396 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -1,9 +1,10 @@ //! Model renames that downstream `__ref` names may have been rewritten for. +use crate::bindings::_helpers::fingerprints::query_hashes::compute_query_hash; use std::collections::BTreeSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_reference_names; @@ -16,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const GENERIC_DIALECT: &str = "generic"; @@ -47,16 +48,9 @@ pub(crate) fn origin_names( .filter(|name| !name.is_empty()) .unwrap_or_else(|| GENERIC_DIALECT.to_owned()); let mut spans = |names: &BTreeSet| ref_name_spans(&query_sql, &dialect_name, names); - let compute_hash: Bound<'_, PyAny> = registry::native(py, NativeAttr::ComputeQueryHash)?; + let dialect_option: Option = dialect.extract()?; let mut query_hash = |text: &str| -> PyResult { - call_with( - &compute_hash, - &[ - ("query_sql", PyString::new(py, text).into_any()), - ("dialect", dialect.clone()), - ], - )? - .extract() + compute_query_hash(py, text, dialect_option.as_deref()) }; origin_reference_names( (&query_sql, recorded_hash), @@ -143,7 +137,7 @@ fn origin_reference_names_of<'py>( /// This plan's renames plus recorded renames, read only when a selected query that references models changed. #[pyfunction] #[pyo3(signature = (**options))] -fn plan_reference_renames<'py>( +pub(crate) fn plan_reference_renames<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 99d1c06bd9..3015e9551c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -2,13 +2,30 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_upstream_deps, +}; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_columns; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use pyo3::PyRef; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -49,17 +66,17 @@ fn clone_plan_output<'py>( exclude: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - let downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; - let selected = native( + let upstream = build_execution_upstream_deps(project)?.into_any(); + let downstream = build_downstream_deps(&upstream)?.into_any(); + let selected = object( py, - NativeAttr::ResolveGraphSelection, - &[ - ("graph", project.getattr("lineage_graph")?), - ("select", select.clone()), - ("exclude", exclude.clone()), - ], + resolve_graph_selection( + project + .getattr("lineage_graph")? + .extract::>()?, + select, + exclude, + )?, )?; planner_model( py, @@ -67,11 +84,7 @@ fn clone_plan_output<'py>( &[ ( "execution_order", - native( - py, - NativeAttr::TopologicallyOrderKeys, - &[("upstream", upstream.clone())], - )?, + object(py, topologically_order_keys(&upstream, None)?)?, ), ("selected_keys", selected), ("upstream_deps", upstream), @@ -95,34 +108,20 @@ fn clone_resolution<'py>( (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = project.py(); - let model_locations = native( - py, - NativeAttr::BuildModelLocations, - &[("models", project.getattr("models")?)], - )?; - let seed_locations = native( - py, - NativeAttr::BuildSeedLocations, - &[("seeds", project.getattr("seeds")?)], - )?; - let function_locations = native( + let model_locations = object(py, build_model_locations(&project.getattr("models")?)?)?; + let seed_locations = object(py, build_seed_locations(&project.getattr("seeds")?)?)?; + let function_locations = object( py, - NativeAttr::BuildFunctionLocations, - &[("functions", project.getattr("functions")?)], + build_function_locations(&project.getattr("functions")?)?, )?; let source_map = PyDict::new(py); for source in source_project.getattr("sources")?.try_iter()? { let source = source?; source_map.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; } - let source_warehouse_columns = native( + let source_warehouse_columns = object( py, - NativeAttr::GatherSourceColumns, - &[ - ("project", source_project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ], + gather_source_columns(source_project, adapter, connection, None)?, )?; Ok(CloneResolution { model_locations, @@ -179,37 +178,31 @@ fn clone_model_entries<'py>( if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; } - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; + let materialization_type = materialization_of(&model)?; let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { let functions = PyDict::new(py); for function in project.getattr("functions")?.try_iter()? { let function = function?; functions.set_item(function.getattr("name")?, function)?; } - let context = native( + let context = Bound::new( py, - RecordId::ModelPlanContext, - &[ - ("model_locations", resolution.model_locations.clone()), - ("models_by_name", PyDict::new(py).into_any()), - ("functions_by_name", functions.into_any()), - ("seed_locations", resolution.seed_locations.clone()), - ("function_locations", resolution.function_locations.clone()), - ("source_map", resolution.source_map.clone()), - ( - "source_warehouse_columns", - resolution.source_warehouse_columns.clone(), - ), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ], - )?; + ModelPlanContext { + model_locations: resolution.model_locations.clone().unbind(), + models_by_name: PyDict::new(py).into_any().unbind(), + functions_by_name: functions.into_any().unbind(), + seed_locations: resolution.seed_locations.clone().unbind(), + function_locations: resolution.function_locations.clone().unbind(), + source_map: resolution.source_map.clone().unbind(), + source_warehouse_columns: resolution.source_warehouse_columns.clone().unbind(), + star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), + future_cursor_config: py.None(), + start_cursor_config: py.None(), + invocation_time: py.None(), + }, + )? + .into_any(); let sql = resolve_model_sql( &Resolution::of_context(adapter.clone(), &context, py.None().into_bound(py))?, &context, @@ -313,25 +306,27 @@ fn clone_function_entries<'py>( if !selected.contains(function.getattr("key")?)? { continue; } - let body_sql = native( + let body_sql = resolve_function_sql( py, - NativeAttr::ResolveFunctionSql, - &[ - ("adapter", adapter.clone()), - ("function", function.clone()), - ("model_locations", resolution.model_locations.clone()), - ("seed_locations", resolution.seed_locations.clone()), - ("function_locations", resolution.function_locations.clone()), - ("source_map", resolution.source_map.clone()), - ( - "source_warehouse_columns", - resolution.source_warehouse_columns.clone(), - ), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ], + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", resolution.model_locations.clone()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?), )?; let field = |name: &str| function.getattr(name); let entry = planner_model( @@ -347,11 +342,7 @@ fn clone_function_entries<'py>( ("body_sql", body_sql), ( "fingerprint_query_sql", - native( - py, - NativeAttr::BuildCompiledFunctionFingerprintSql, - &[("function", function.clone())], - )?, + object(py, build_compiled_function_fingerprint_sql(&function)?)?, ), ("fingerprint_destination", field("fingerprint_destination")?), ("return_columns", field("return_columns")?), @@ -416,11 +407,7 @@ fn origin_model_entries<'py>( if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; } - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; + let materialization_type = materialization_of(&model)?; entries.push(model_entry(&model, materialization_type, string(py, ""))?); } tuple_of(py, entries) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 1930333df0..041d7a9afd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -2,15 +2,24 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_entries::values::{ - boolean, native, none, planner_model, planner_type, + boolean, none, planner_model, planner_type, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_identity::metadata::build_seed_identity; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -82,21 +91,14 @@ fn resolve_static_command_scope<'py>( auto_load_sources: bool, ) -> PyResult> { let py = project.py(); - native( - py, - NativeAttr::ResolvePlannerScopes, - &[ - ("project", project.clone()), - ("selection", selection.clone()), - ( - "policies", - planner_model( - py, - RecordId::PlannerPolicies, - &[("auto_load_sources", boolean(py, auto_load_sources)?)], - )?, - ), - ], + resolve_planner_scopes( + project, + selection, + &planner_model( + py, + RecordId::PlannerPolicies, + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, )? .getattr("selected_scope") } @@ -150,24 +152,21 @@ fn build_audit_command_plan<'py>( let model = model?; materializations.set_item( model_reads::name_object(&model)?, - native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )? - .getattr("value")?, + materialization_of(&model)?.getattr("value")?, )?; } - let audits = native( + let audits = object( py, - NativeAttr::BuildSelectedAuditEntries, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, )?; base_plan_output( project, @@ -235,11 +234,7 @@ fn build_test_command_plan<'py>( if !function_keys.contains(function.getattr("key")?)? { continue; } - let fingerprint_sql = native( - py, - NativeAttr::BuildCompiledFunctionFingerprintSql, - &[("function", function.clone())], - )?; + let fingerprint_sql = object(py, build_compiled_function_fingerprint_sql(&function)?)?; functions.push(function_entry( &function, &adapter, @@ -299,7 +294,7 @@ fn build_seed_command_plan<'py>( if !selected_keys.contains(seed.getattr("key")?)? { continue; } - let identity = native(py, NativeAttr::BuildSeedIdentity, &[("seed", seed.clone())])?; + let identity = object(py, build_seed_identity(&seed)?)?; let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); let previous = existing.call_method1("get", (seed.getattr("name")?,))?; let reason = if previous.is_none() { @@ -320,14 +315,7 @@ fn build_seed_command_plan<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - native( - py, - NativeAttr::ExtractSeedColumns, - &[("seed", seed.clone())], - )?, - ), + ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -353,7 +341,7 @@ fn build_seed_command_plan<'py>( /// Project static relation data for Python checks. #[pyfunction] #[pyo3(signature = (*, project, scope, relations))] -fn build_relation_command_plan<'py>( +pub(crate) fn build_relation_command_plan<'py>( project: &Bound<'py, PyAny>, scope: &Bound<'py, PyAny>, relations: &Bound<'py, PyAny>, @@ -415,9 +403,8 @@ fn read_selected_seed_fingerprints<'py>( } } let node_names = PyTuple::new(py, names.iter())?; - let latest = native( + let latest = read_latest_fingerprints(Some(&keywords( py, - NativeAttr::ReadLatestFingerprints, &[ ("connection", connection.clone()), ("execute", adapter.getattr("execute")?), @@ -439,7 +426,7 @@ fn read_selected_seed_fingerprints<'py>( ), ("node_names", node_names.into_any()), ], - )?; + )?))?; let items = PyList::new( py, latest diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs new file mode 100644 index 0000000000..44111a26d4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/contexts.rs @@ -0,0 +1,171 @@ +//! Relation, model-plan and entry-build contexts: built from keyword arguments and read by plan entry construction. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::{PyDict, PyFrozenSet}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::{ + ModelPlanContext, PlanEntryBuildInputs, PlannerRelationsContext, +}; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; + +/// The argument `name`, or `default()` when it was not passed. +fn value_or<'py>( + args: &KeywordArguments<'py>, + name: &str, + default: impl FnOnce() -> PyResult>, +) -> PyResult> { + Ok(if args.passed(name)? { + args.value(name)? + } else { + default()? + } + .unbind()) +} + +fn empty_frozenset(py: Python<'_>) -> PyResult> { + Ok(PyFrozenSet::empty(py)?.into_any()) +} + +#[pymethods] +impl PlannerRelationsContext { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerRelationsContext", + kwargs, + ( + &[ + "model_locations", + "seed_locations", + "function_locations", + "source_map", + "source_read_map", + "python_source_read_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &["listed_source_names"], + ), + )?; + Ok(Self { + model_locations: args.value("model_locations")?.unbind(), + seed_locations: args.value("seed_locations")?.unbind(), + function_locations: args.value("function_locations")?.unbind(), + source_map: args.value("source_map")?.unbind(), + source_read_map: args.value("source_read_map")?.unbind(), + python_source_read_map: args.value("python_source_read_map")?.unbind(), + source_warehouse_columns: args.value("source_warehouse_columns")?.unbind(), + star_exclude_keyword: args.value("star_exclude_keyword")?.unbind(), + listed_source_names: value_or(&args, "listed_source_names", || empty_frozenset(py))?, + }) + } +} + +#[pymethods] +impl ModelPlanContext { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "ModelPlanContext", + kwargs, + ( + &[ + "model_locations", + "models_by_name", + "seed_locations", + "function_locations", + "source_map", + "source_warehouse_columns", + "star_exclude_keyword", + ], + &[ + "functions_by_name", + "runtime_cursor_producer_names", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + ], + ), + )?; + Ok(Self { + model_locations: args.value("model_locations")?.unbind(), + models_by_name: args.value("models_by_name")?.unbind(), + seed_locations: args.value("seed_locations")?.unbind(), + function_locations: args.value("function_locations")?.unbind(), + source_map: args.value("source_map")?.unbind(), + source_warehouse_columns: args.value("source_warehouse_columns")?.unbind(), + star_exclude_keyword: args.value("star_exclude_keyword")?.unbind(), + functions_by_name: value_or(&args, "functions_by_name", || { + Ok(PyDict::new(py).into_any()) + })?, + runtime_cursor_producer_names: value_or( + &args, + "runtime_cursor_producer_names", + || empty_frozenset(py), + )?, + future_cursor_config: args.value("future_cursor_config")?.unbind(), + start_cursor_config: args.value("start_cursor_config")?.unbind(), + invocation_time: args.value("invocation_time")?.unbind(), + }) + } +} + +#[pymethods] +impl PlanEntryBuildInputs { + #[new] + #[pyo3(signature = (**kwargs))] + pub(crate) fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlanEntryBuildInputs", + kwargs, + ( + &[], + &[ + "source_freshness_blocked_model_names", + "external_blocked_model_names", + "start_cursor_override", + "end_cursor_override", + "future_cursor_config", + "start_cursor_config", + "invocation_time", + "max_microbatches", + "max_microbatches_is_override", + "microbatch_limit_action", + ], + ), + )?; + Ok(Self { + source_freshness_blocked_model_names: value_or( + &args, + "source_freshness_blocked_model_names", + || empty_frozenset(py), + )?, + external_blocked_model_names: value_or(&args, "external_blocked_model_names", || { + empty_frozenset(py) + })?, + start_cursor_override: args.value("start_cursor_override")?.unbind(), + end_cursor_override: args.value("end_cursor_override")?.unbind(), + future_cursor_config: args.value("future_cursor_config")?.unbind(), + start_cursor_config: args.value("start_cursor_config")?.unbind(), + invocation_time: args.value("invocation_time")?.unbind(), + max_microbatches: args.value("max_microbatches")?.unbind(), + max_microbatches_is_override: args.flag("max_microbatches_is_override")?, + microbatch_limit_action: value_or(&args, "microbatch_limit_action", || { + default_microbatch_limit_action(py) + })?, + }) + } +} + +/// The microbatch limit action used when a project configures none. +pub(crate) fn default_microbatch_limit_action(py: Python<'_>) -> PyResult> { + registry::class(py, RecordId::MicrobatchLimitAction)?.getattr("ERROR") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index b56f6ed178..48509ee8c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -5,8 +5,13 @@ use crate::bindings::_helpers::model_config::config_values::config_str as author use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_upstream_deps, +}; +use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_strategy::strategy::materialization_of; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -46,18 +51,10 @@ fn display_entry<'py>( full_refresh: bool, ) -> PyResult> { let py = model.py(); - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; - let effective = native( + let materialization_type = materialization_of(model)?; + let effective = object( py, - NativeAttr::ResolveModelFullRefresh, - &[ - ("model", model.clone()), - ("cli_full_refresh", boolean(py, full_refresh)?), - ], + resolve_model_full_refresh(model, boolean(py, full_refresh)?.is_truthy()?)?, )? .is_truthy()?; let query_sql = model_reads::query_sql(model)?; @@ -151,9 +148,8 @@ fn build_display_only_sqlbuild_plan<'py>( .iter() .map(|entry| entry.getattr("key")) .collect::>()?; - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - let downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; + let upstream = build_execution_upstream_deps(project)?.into_any(); + let downstream = build_downstream_deps(&upstream)?.into_any(); planner_model( py, RecordId::PlanOutput, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 74cbae755f..4cf714f6d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -13,13 +13,16 @@ use crate::bindings::_helpers::planner_entries::test_planning::{ }; use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::execution_changes::{ build_planner_entry_results, reconcile_execution_changes, }; use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, }; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; @@ -46,7 +49,7 @@ fn scopes_and_cursor_overrides<'py>( &args.value("selection")?, &args.value("policies")?, )?; - let selected = scopes.getattr("selected_scope")?; + let selected = class_field(&scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let models_by_name = selected.getattr("models_by_name")?; let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; let mut models: Vec> = Vec::new(); @@ -73,9 +76,15 @@ fn change_results<'py>( policies: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); - let versions = identities.getattr("version_identities")?; + let versions = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; let changes = detect_changes( - (project, &scopes.getattr("inspection_scope")?, snapshot), + ( + project, + &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + snapshot, + ), overrides.getattr("full_refresh")?.is_truthy()?, ( versions.getattr("model_version_hashes")?, @@ -123,7 +132,7 @@ fn planned_output<'py>( overrides, &policies, )?; - let inspection_scope = scopes.getattr("inspection_scope")?; + let inspection_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; let source_freshness = build_planner_source_freshness_result( (&project, &inspection_scope), @@ -174,7 +183,7 @@ fn planned_output<'py>( /// Build the execution plan for one selection against live warehouse state. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_execution_plan<'py>( +pub(crate) fn build_execution_plan<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -212,7 +221,7 @@ fn build_execution_plan<'py>( )? .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; - let inspection = scopes.getattr("inspection_scope")?; + let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; let test_planning = start_planning( py, PlanningRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 519d1ac918..a8dddfd37f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,9 +1,10 @@ //! Plan entry and plan output assembly over native planner resolution. mod clone_plans; -mod command_plans; +pub(crate) mod command_plans; +pub(crate) mod contexts; mod display_plan; -mod execution_plan; +pub(crate) mod execution_plan; pub(crate) mod model_entries; pub(crate) mod models; pub(crate) mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 5b9898a4b9..63889b31af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,17 +1,24 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; -use crate::bindings::_helpers::planner_entries::models::PlanEntriesRequest; +use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; +use crate::bindings::_helpers::planner_entries::models::{ + ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, +}; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, }; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -21,6 +28,7 @@ use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as Sq use crate::bindings::_helpers::planner_resolution::orchestration::{ Resolution, model_hook_entries, resolve_model_sql, }; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::runtime_cursor_producer_names; use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ build_cursor_input_relations, reject_unexposed_cursor_input_columns, }; @@ -34,9 +42,10 @@ use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; use crate::bindings::_helpers::planner_strategy::values::text; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::bindings::_helpers::semantic_shapes::source_bindings::source_binding_diagnostics; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -56,18 +65,21 @@ impl<'py> ContextReads<'py> { adapter: Bound<'py, PyAny>, external_resolver: Bound<'py, PyAny>, ) -> PyResult { + let py = context.py(); + let fields = context.cast::()?.get(); + let field = |value: &Py| value.bind(py).clone(); Ok(Self { lookups: CursorInputLookups { - models_by_name: context.getattr("models_by_name")?, - functions_by_name: context.getattr("functions_by_name")?, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + models_by_name: field(&fields.models_by_name), + functions_by_name: field(&fields.functions_by_name), + source_map: field(&fields.source_map), + source_warehouse_columns: field(&fields.source_warehouse_columns), + model_locations: field(&fields.model_locations), + seed_locations: field(&fields.seed_locations), + runtime_cursor_producer_names: field(&fields.runtime_cursor_producer_names), }, resolution: Resolution::of_context(adapter, &context, external_resolver)?, - invocation_time: context.getattr("invocation_time")?, + invocation_time: field(&fields.invocation_time), context, }) } @@ -222,18 +234,25 @@ fn check_binding_diagnostics<'py>( errors.push(diagnostic); } } - let bound = native( + let bound = object( py, - NativeAttr::SourceBindingDiagnostics, - &[ - ("project", project.clone()), - ("columns", relations.getattr("source_warehouse_columns")?), - ( - "profile", - adapter.call_method0("expression_inference_profile")?, - ), - ("selected_keys", selected_keys.clone()), - ], + source_binding_diagnostics(Some(&keywords( + py, + &[ + ("project", project.clone()), + ( + "columns", + class_field(relations, |context: &PlannerRelationsContext| { + &context.source_warehouse_columns + })?, + ), + ( + "profile", + adapter.call_method0("expression_inference_profile")?, + ), + ("selected_keys", selected_keys.clone()), + ], + )?))?, )?; for diagnostic in bound.try_iter()? { let diagnostic = diagnostic?; @@ -296,7 +315,7 @@ fn check_microbatch_concurrency<'py>( } let settings = project.getattr("settings")?; if !configured.is_empty() && !settings.getattr("microbatch_concurrency")?.is_truthy()? { - let guidance = native(py, NativeAttr::MicrobatchGuidance, &[])?; + let guidance = object(py, microbatch_guidance())?; let prefix: &str = MICROBATCH_CONCURRENCY_MESSAGE; return Err(error_of( py, @@ -385,23 +404,21 @@ pub(crate) fn plan_entries<'py>( } = request; let py = project.py(); let inputs = if inputs.is_none() { - native(py, RecordId::PlanEntryBuildInputs, &[])? + Bound::new(py, PlanEntryBuildInputs::new(py, None)?)?.into_any() } else { inputs }; + let build = inputs.cast::()?.get(); + let relations_context = relations.cast::()?.get(); let selected_keys = scope.getattr("selected_keys")?; check_binding_diagnostics(&project, &adapter, &relations, &selected_keys)?; - let freshness_blocked = inputs.getattr("source_freshness_blocked_model_names")?; - let external_blocked = inputs.getattr("external_blocked_model_names")?; - let start_override = inputs.getattr("start_cursor_override")?; - let end_override = inputs.getattr("end_cursor_override")?; - let producers = native( + let freshness_blocked = build.source_freshness_blocked_model_names.bind(py).clone(); + let external_blocked = build.external_blocked_model_names.bind(py).clone(); + let start_override = build.start_cursor_override.bind(py).clone(); + let end_override = build.end_cursor_override.bind(py).clone(); + let producers = object( py, - NativeAttr::RuntimeCursorProducerNames, - &[ - ("scope", scope.clone()), - ("source_map", relations.getattr("source_map")?), - ], + runtime_cursor_producer_names(&scope, &relations_context.source_map.bind(py).clone())?, )?; let functions_by_name = PyDict::new(py); for function in project.getattr("functions")?.try_iter()? { @@ -410,39 +427,24 @@ pub(crate) fn plan_entries<'py>( } let settings = project.getattr("settings")?; let models_by_name = scope.getattr("models_by_name")?; - let context = native( + let context = Bound::new( py, - RecordId::ModelPlanContext, - &[ - ("model_locations", relations.getattr("model_locations")?), - ("models_by_name", models_by_name.clone()), - ("functions_by_name", functions_by_name.into_any()), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", relations.getattr("source_read_map")?), - ( - "source_warehouse_columns", - relations.getattr("source_warehouse_columns")?, - ), - ( - "star_exclude_keyword", - relations.getattr("star_exclude_keyword")?, - ), - ("runtime_cursor_producer_names", producers), - ( - "future_cursor_config", - inputs.getattr("future_cursor_config")?, - ), - ( - "start_cursor_config", - inputs.getattr("start_cursor_config")?, - ), - ("invocation_time", inputs.getattr("invocation_time")?), - ], - )?; + ModelPlanContext { + model_locations: relations_context.model_locations.clone_ref(py), + models_by_name: models_by_name.clone().unbind(), + functions_by_name: functions_by_name.into_any().unbind(), + seed_locations: relations_context.seed_locations.clone_ref(py), + function_locations: relations_context.function_locations.clone_ref(py), + source_map: relations_context.source_read_map.clone_ref(py), + source_warehouse_columns: relations_context.source_warehouse_columns.clone_ref(py), + star_exclude_keyword: relations_context.star_exclude_keyword.clone_ref(py), + runtime_cursor_producer_names: producers.unbind(), + future_cursor_config: build.future_cursor_config.clone_ref(py), + start_cursor_config: build.start_cursor_config.clone_ref(py), + invocation_time: build.invocation_time.clone_ref(py), + }, + )? + .into_any(); let reads = ContextReads::of( context, adapter.clone(), @@ -536,7 +538,7 @@ pub(crate) fn plan_entries<'py>( /// Detect changes and build a plan entry and warnings for a single model. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn plan_model<'py>( +pub(crate) fn plan_model<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -560,19 +562,21 @@ fn plan_model<'py>( )?; let model = args.value("model")?; let snapshot = args.value("snapshot")?; - let change_result = native( + let change_result = detect_model_changes( py, - NativeAttr::DetectModelChanges, - &[ - ("model", model.clone()), - ("snapshot", snapshot.clone()), - ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), - ( - "query_change_tracking", - args.value("query_change_tracking")?, - ), - ("full_refresh", args.value("full_refresh")?), - ], + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?), )?; let overrides = args.value("cursor_overrides")?; plan_model_from_change(&ModelPlanRequest { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index 0cc640c9cb..6692769335 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -1,6 +1,7 @@ //! Typed requests of plan entry construction. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; /// What resolving relation locations and source metadata for one scope reads. #[derive(Debug)] @@ -44,3 +45,52 @@ pub(crate) struct PlanOutputInputs<'py> { pub(crate) reload_sources: Bound<'py, PyAny>, pub(crate) extras: Bound<'py, PyAny>, } + +/// Resolved relation and source inputs for plan entry construction. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerRelationsContext { + pub(crate) model_locations: Py, + pub(crate) seed_locations: Py, + pub(crate) function_locations: Py, + pub(crate) source_map: Py, + pub(crate) source_read_map: Py, + pub(crate) python_source_read_map: Py, + pub(crate) source_warehouse_columns: Py, + pub(crate) star_exclude_keyword: Py, + pub(crate) listed_source_names: Py, +} + +/// Relation lookups and source columns for building one model plan entry. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct ModelPlanContext { + pub(crate) model_locations: Py, + pub(crate) models_by_name: Py, + pub(crate) seed_locations: Py, + pub(crate) function_locations: Py, + pub(crate) source_map: Py, + pub(crate) source_warehouse_columns: Py, + pub(crate) star_exclude_keyword: Py, + pub(crate) functions_by_name: Py, + pub(crate) runtime_cursor_producer_names: Py, + pub(crate) future_cursor_config: Py, + pub(crate) start_cursor_config: Py, + pub(crate) invocation_time: Py, +} + +/// Blocked models and cursor overrides for plan entry building. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlanEntryBuildInputs { + pub(crate) source_freshness_blocked_model_names: Py, + pub(crate) external_blocked_model_names: Py, + pub(crate) start_cursor_override: Py, + pub(crate) end_cursor_override: Py, + pub(crate) future_cursor_config: Py, + pub(crate) start_cursor_config: Py, + pub(crate) invocation_time: Py, + pub(crate) max_microbatches: Py, + pub(crate) max_microbatches_is_override: bool, + pub(crate) microbatch_limit_action: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index d4f97a8f78..e4b9d2fd82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,13 +1,20 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; +use crate::bindings::_helpers::planner_scope::loaders::upstream_loader_dependency_names; +use crate::bindings::_helpers::planner_scope::sources::build_source_load_entries; +use crate::bindings::_helpers::planner_strategy::audit_entries::build_selected_audit_entries; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; @@ -119,14 +126,7 @@ fn seed_entries<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - native( - py, - NativeAttr::ExtractSeedColumns, - &[("seed", seed.clone())], - )?, - ), + ("columns", object(py, extract_seed_columns(&seed)?)?), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -290,13 +290,9 @@ fn require_intermediate_loader_targets<'py>( else { continue; }; - let dependencies = native( + let dependencies = object( py, - NativeAttr::UpstreamLoaderDependencyNames, - &[ - ("loader_function", loader_function), - ("loader_functions", loader_functions.clone()), - ], + upstream_loader_dependency_names(&loader_function, &loader_functions)?, )?; for dependency_name in dependencies.try_iter()? { let dependency_name = dependency_name?; @@ -343,28 +339,30 @@ pub(super) fn function_entry<'py>( ), ) -> PyResult> { let py = function.py(); - let body_sql = native( + let body_sql = resolve_function_sql( py, - NativeAttr::ResolveFunctionSql, - &[ - ("adapter", adapter.clone()), - ("function", function.clone()), - ("model_locations", relations.getattr("model_locations")?), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", relations.getattr("source_read_map")?), - ( - "source_warehouse_columns", - relations.getattr("source_warehouse_columns")?, - ), - ( - "star_exclude_keyword", - relations.getattr("star_exclude_keyword")?, - ), - ], + Some(keywords( + py, + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ], + )?), )?; let field = |name: &str| function.getattr(name); planner_model( @@ -565,15 +563,14 @@ pub(crate) fn plan_output_fields<'py>( let execution_order = scope.getattr("execution_order")?; let source_map = relations.getattr("source_map")?; let seeds = seed_entries(&project, &snapshot, &selected_keys, &extras)?; - let source_load_entries = native( + let source_load_entries = object( py, - NativeAttr::BuildSourceLoadEntries, - &[ - ("execution_order", execution_order.clone()), - ("selected_keys", selected_keys.clone()), - ("source_map", source_map.clone()), - ("is_reload", reload_sources), - ], + build_source_load_entries( + &execution_order, + &selected_keys, + source_map.downcast::()?, + reload_sources.is_truthy()?, + )?, )?; require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; let functions = function_entries( @@ -589,16 +586,18 @@ pub(crate) fn plan_output_fields<'py>( entry.getattr("materialization_type")?, )?; } - let audit_entries = native( + let audit_entries = object( py, - NativeAttr::BuildSelectedAuditEntries, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("scope", scope.clone()), - ("relations", relations.clone()), - ("model_materializations", materializations.into_any()), - ], + build_selected_audit_entries(Some(&keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?))?, )?; let selection = scope.getattr("sql_test_selection")?; let mut planned = extras.getattr("planned_sql_tests")?; @@ -717,7 +716,7 @@ fn plan_function<'py>( /// Plan SQL tests selected by name or overlapping one selection into a reusable result. #[pyfunction] #[pyo3(signature = (*, project, adapter, selected_keys, sql_test_selection))] -fn plan_selected_sql_tests<'py>( +pub(crate) fn plan_selected_sql_tests<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, selected_keys: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index 15f31ed296..ed45e8827f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -1,8 +1,11 @@ //! Register plan entry and plan output assembly entry points. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; super::clone_plans::register(module)?; super::command_plans::register(module)?; super::display_plan::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 7a2e29c8e4..d67d2d00b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,10 +1,23 @@ //! Relation locations and source metadata for plan entry construction. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::planner_entries::models::RelationsRequest; -use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_entries::models::{ + PlannerRelationsContext, RelationsRequest, +}; +use crate::bindings::_helpers::planner_entries::values::{none, planner_model}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_model_locations; +use crate::bindings::_helpers::planner_resolution::locations::build_seed_locations; +use crate::bindings::_helpers::planner_resolution::refs::apply_deferred_locations; +use crate::bindings::_helpers::planner_scope::sources::build_source_load_map; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_warehouse::semantic_sources::semantic_source_columns; +use crate::bindings::_helpers::planner_warehouse::source_columns::gather_source_inspection; +use crate::bindings::_helpers::planner_warehouse::source_deferral::build_source_read_map; +use crate::bindings::_helpers::planner_warehouse::source_deferral::with_declared_source_reads; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -52,15 +65,11 @@ fn source_warehouse_columns<'py>( .try_iter()? .collect::>>()?, )?; - let gathered = native( - py, - NativeAttr::GatherSourceInspection, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("source_entries", entries.into_any()), - ], + let gathered = gather_source_inspection( + project, + adapter, + connection, + entries.into_any().extract::>>()?, )?; Ok((gathered.get_item(0)?, gathered.get_item(1)?)) } @@ -87,32 +96,22 @@ pub(crate) fn planner_relations_context<'py>( deferral }; let selected_keys = scope.getattr("selected_keys")?; - let mut model_locations = native( + let mut model_locations = object(py, build_model_locations(&project.getattr("models")?)?)?; + let mut seed_locations = object(py, build_seed_locations(&project.getattr("seeds")?)?)?; + let function_locations = object( py, - NativeAttr::BuildModelLocations, - &[("models", project.getattr("models")?)], - )?; - let mut seed_locations = native( - py, - NativeAttr::BuildSeedLocations, - &[("seeds", project.getattr("seeds")?)], - )?; - let function_locations = native( - py, - NativeAttr::BuildFunctionLocations, - &[("functions", project.getattr("functions")?)], + build_function_locations(&project.getattr("functions")?)?, )?; let deferred_locations = deferral.getattr("deferred_locations")?; if !deferred_locations.is_none() { - let applied = native( + let applied = object( py, - NativeAttr::ApplyDeferredLocations, - &[ - ("model_locations", model_locations), - ("seed_locations", seed_locations), - ("deferred_locations", deferred_locations), - ("selected_keys", selected_keys.clone()), - ], + apply_deferred_locations( + model_locations.downcast::()?, + seed_locations.downcast::()?, + deferred_locations.downcast::()?, + &selected_keys, + )?, )?; model_locations = applied.get_item(0)?; seed_locations = applied.get_item(1)?; @@ -123,24 +122,23 @@ pub(crate) fn planner_relations_context<'py>( } else { relation_keys }; - let source_map = native( + let source_map = object( py, - NativeAttr::BuildSourceLoadMap, - &[ - ("project", project.clone()), - ("selected_keys", load_keys), - ( - "upstream_deps", - if scope.getattr("all_keys")?.is_truthy()? { - scope.getattr("upstream_deps")? - } else { - none(py) - }, - ), - ], + build_source_load_map( + &project, + &load_keys, + Some(&if scope.getattr("all_keys")?.is_truthy()? { + scope.getattr("upstream_deps")? + } else { + none(py) + }) + .filter(|value| !value.is_none()), + )?, )?; let deferral_enabled = deferral.getattr("source_deferral_enabled")?.is_truthy()?; - let read_map = |map: &Bound<'py, PyAny>, names: Option>| { + let read_map = |map: &Bound<'py, PyAny>, + names: Option>| + -> PyResult> { if !deferral_enabled { return Ok(map.clone()); } @@ -155,17 +153,16 @@ pub(crate) fn planner_relations_context<'py>( if let Some(names) = names { pairs.push(("read_source_names", names)); } - native(py, NativeAttr::BuildSourceReadMap, &pairs) + Ok(build_source_read_map(py, Some(keywords(py, &pairs)?))?.into_any()) }; let source_read_map = read_map(&source_map, None)?; - let python_source_map = native( + let python_source_map = object( py, - NativeAttr::WithDeclaredSourceReads, - &[ - ("project", project.clone()), - ("source_map", source_map.clone()), - ("source_names", scope.getattr("python_read_source_names")?), - ], + with_declared_source_reads( + &project, + source_map.downcast::()?, + &scope.getattr("python_read_source_names")?, + )?, )?; let python_names = PyFrozenSet::new( py, @@ -180,36 +177,36 @@ pub(crate) fn planner_relations_context<'py>( &source_read_map, &known_source_columns, )?; - let semantic_columns = native( + let semantic_columns = semantic_source_columns( py, - NativeAttr::GetSemanticSourceColumns, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection), - ("source_read_map", source_read_map.clone()), - ("selected_keys", selected_keys), - ("columns", columns), - ], - )?; - planner_model( + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection), + ("source_read_map", source_read_map.clone()), + ("selected_keys", selected_keys), + ("columns", columns), + ], + )?), + )? + .into_any(); + Ok(Bound::new( py, - RecordId::PlannerRelationsContext, - &[ - ("model_locations", model_locations), - ("seed_locations", seed_locations), - ("function_locations", function_locations), - ("source_map", source_map), - ("source_read_map", source_read_map), - ("python_source_read_map", python_source_read_map), - ("source_warehouse_columns", semantic_columns), - ( - "star_exclude_keyword", - adapter.call_method0("star_exclude_keyword")?, - ), - ("listed_source_names", listed_source_names), - ], - ) + PlannerRelationsContext { + model_locations: model_locations.unbind(), + seed_locations: seed_locations.unbind(), + function_locations: function_locations.unbind(), + source_map: source_map.unbind(), + source_read_map: source_read_map.unbind(), + python_source_read_map: python_source_read_map.unbind(), + source_warehouse_columns: semantic_columns.unbind(), + star_exclude_keyword: adapter.call_method0("star_exclude_keyword")?.unbind(), + listed_source_names: listed_source_names.unbind(), + }, + )? + .into_any()) } /// Resolve relation locations and source metadata for plan entry construction. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 2b47455ce3..6335cc5e60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -80,7 +80,9 @@ fn hook_label(hook_name: &str, kind: &str) -> String { /// Execution-order edges from lineage and woven SQL test nodes. #[pyfunction] -fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_execution_upstream_deps<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project.py(); let upstream: Bound<'py, PyDict> = PyDict::new(py); for collection in ["models", "sources", "seeds", "functions"] { @@ -162,7 +164,9 @@ fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult(project: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_execution_edge_origins<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = project.py(); let origins: Bound<'py, PyDict> = PyDict::new(py); for test in project.getattr("sql_tests")?.try_iter()? { @@ -219,7 +223,9 @@ fn build_execution_edge_origins<'py>(project: &Bound<'py, PyAny>) -> PyResult(upstream: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_downstream_deps<'py>( + upstream: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = upstream.py(); let result: Bound<'py, PyDict> = PyDict::new(py); for (key, sources) in inverted(upstream)?.iter() { @@ -265,7 +271,7 @@ fn edge_origins<'py>( /// All keys in stable dependency order using Kahn's algorithm. #[pyfunction] #[pyo3(signature = (*, upstream, injected_edge_origins = None))] -fn topologically_order_keys<'py>( +pub(crate) fn topologically_order_keys<'py>( upstream: &Bound<'py, PyAny>, injected_edge_origins: Option>, ) -> PyResult> { @@ -327,7 +333,7 @@ fn missing_rows<'py>( /// Upstream deps of selected keys that exist neither in scope nor in the warehouse. #[pyfunction] #[pyo3(signature = (**options))] -fn check_buildability<'py>( +pub(crate) fn check_buildability<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -386,7 +392,7 @@ fn check_buildability<'py>( /// Describe missing upstream keys, naming the audit when only an audit read requires one. #[pyfunction] #[pyo3(signature = (*, missing, edge_origins))] -fn missing_upstream_message<'py>( +pub(crate) fn missing_upstream_message<'py>( missing: &Bound<'py, PyAny>, edge_origins: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs index ca508ec8b1..07a5cc44d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -75,7 +75,7 @@ pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { /// Fingerprint a model query for change detection, naming the model when it cannot. #[pyfunction] #[pyo3(signature = (*, model_name, query_sql, dialect))] -fn model_definition_hash( +pub(crate) fn model_definition_hash( py: Python<'_>, model_name: &str, query_sql: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 9ecd1ba24a..5f93834e0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -64,7 +64,7 @@ fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult( +pub(crate) fn build_direct_model_version_identities<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -201,7 +201,7 @@ fn build_direct_model_version_identities<'py>( /// Planned write hashes replaced by hashes based on upstreams available this run. #[pyfunction] #[pyo3(signature = (**options))] -fn with_honest_model_write_hashes<'py>( +pub(crate) fn with_honest_model_write_hashes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -354,7 +354,7 @@ fn with_honest_model_write_hashes<'py>( /// Resolved actions with recomputed model changes patched in where available. #[pyfunction] #[pyo3(signature = (*, resolved_actions, changes))] -fn merge_recomputed_model_changes<'py>( +pub(crate) fn merge_recomputed_model_changes<'py>( resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 35d06a6406..a909523176 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -456,7 +456,7 @@ fn build_version_identity_metadata_json<'py>( /// Deterministic non-query model fingerprint metadata JSON. #[pyfunction] #[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] -fn build_model_version_identity_metadata_json<'py>( +pub(crate) fn build_model_version_identity_metadata_json<'py>( model: &Bound<'py, PyAny>, function_local_hashes: Option>, hook_version_hashes: Option>, @@ -478,7 +478,7 @@ fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { /// Local-only semantic hashes of functions by name. #[pyfunction] #[pyo3(signature = (*, functions, dialect))] -fn build_function_local_hashes<'py>( +pub(crate) fn build_function_local_hashes<'py>( functions: &Bound<'py, PyAny>, dialect: Option<&str>, ) -> PyResult> { @@ -487,13 +487,15 @@ fn build_function_local_hashes<'py>( /// Stable function definition text used for change fingerprints. #[pyfunction] -fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn build_compiled_function_fingerprint_sql( + function: &Bound<'_, PyAny>, +) -> PyResult { compiled_function_fingerprint_sql(function) } /// `(identity_hash, metadata_json)` for one compiled seed. #[pyfunction] -fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { +pub(crate) fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { seed_identity(seed) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index 5559b2c1e6..716f47a687 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -170,7 +170,7 @@ fn plan_warnings<'py>( /// Warn for selected models stale through changed upstreams outside the run set. #[pyfunction] #[pyo3(signature = (**options))] -fn build_stale_out_of_selection_warnings<'py>( +pub(crate) fn build_stale_out_of_selection_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index 8d4f973716..4299bea43d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -2,12 +2,12 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, native, + config, definition_hash, lower, }; use crate::bindings::_helpers::planner_migrations::models::{ ColumnCandidate, ColumnMigrationInputs, MigrationPhase, }; -use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::planner_migrations::projections::parse_query_shape; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -73,24 +73,18 @@ pub(super) fn candidates<'py>( by_name.set_item(lower(&column.getattr("name")?)?, column)?; } let previous = if changed { - native( + parse_query_shape( py, - NativeAttr::ParseQueryShape, - &[ - ("query_sql", fingerprint.getattr("definition")?), - ("dialect", dialect.clone()), - ], + &fingerprint.getattr("definition")?.extract::()?, + dialect.extract::>()?.as_deref(), )? } else { py.None().into_bound(py) }; - let current = native( + let current = parse_query_shape( py, - NativeAttr::ParseQueryShape, - &[ - ("query_sql", model_reads::query_sql(&model)?), - ("dialect", dialect.clone()), - ], + &model_reads::query_sql(&model)?.extract::()?, + dialect.extract::>()?.as_deref(), )?; result.push(ColumnCandidate { model, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 668ac1047d..128f2c6273 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -3,16 +3,18 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_candidates::declared_names; -use crate::bindings::_helpers::planner_migrations::column_objects::{ - lower, model, native, replace, -}; +use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace}; +use crate::bindings::_helpers::planner_migrations::columns::identical_renames; +use crate::bindings::_helpers::planner_migrations::columns::rename_hints; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_column; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -33,16 +35,21 @@ pub(super) fn decide<'py>( pairs.push((origin, destination, "manual")); } if !candidate.previous.is_none() && !candidate.current.is_none() { - let detected = native( + let detected = object( py, - NativeAttr::IdenticalRenames, - &[ - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("excluded", declared_names(candidate)?.into_any()), - ("declared", declared.into_any()), - ("input_columns", inputs.clone()), - ], + identical_renames( + py, + Some(keywords( + py, + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", declared_names(candidate)?.into_any()), + ("declared", declared.into_any()), + ("input_columns", inputs.clone()), + ], + )?), + )?, )?; for pair in detected.try_iter()? { let pair = pair?; @@ -226,18 +233,23 @@ pub(super) fn hints<'py>( claimed.set_item(lower(&entry.getattr("origin_column")?)?, py.None())?; claimed.set_item(lower(&entry.getattr("destination_column")?)?, py.None())?; } - native( + object( py, - NativeAttr::RenameHints, - &[ - ("model_name", model_reads::name_object(&candidate.model)?), - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("excluded", excluded.call_method1("union", (claimed,))?), - ( - "live_columns", - PyFrozenSet::new(py, candidate.warehouse.keys().iter())?.into_any(), - ), - ], + rename_hints( + py, + Some(keywords( + py, + &[ + ("model_name", model_reads::name_object(&candidate.model)?), + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("excluded", excluded.call_method1("union", (claimed,))?), + ( + "live_columns", + PyFrozenSet::new(py, candidate.warehouse.keys().iter())?.into_any(), + ), + ], + )?), + )?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index e7f00d1aa4..333e6dc940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -3,7 +3,9 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; -use crate::bindings::_helpers::records::ids::{NativeAttr, NativeItem, RecordId}; +use crate::bindings::_helpers::planner_identity::hashing::model_definition_hash; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::{NativeItem, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; @@ -54,13 +56,16 @@ pub(super) fn definition_hash<'py>( model: &Bound<'py, PyAny>, snapshot: &Bound<'py, PyAny>, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::ModelDefinitionHash, - &[ - ("model_name", model_reads::name_object(model)?), - ("query_sql", model_reads::query_sql(model)?), - ("dialect", snapshot.getattr("column_dialect")?), - ], + model_definition_hash( + model.py(), + &model_reads::name_object(model)?.extract::()?, + &model_reads::query_sql(model)?.extract::()?, + snapshot + .getattr("column_dialect")? + .extract::>()? + .as_deref(), + )?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 1126347f21..936d5c1fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,7 +1,8 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; @@ -9,8 +10,9 @@ use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_migrations::projections::renames_explain_change; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -68,15 +70,20 @@ pub(super) fn overlay<'py>( if !candidate.fingerprint.is_none() && !candidate.previous.is_none() && !candidate.current.is_none() - && native( + && object( py, - NativeAttr::RenamesExplainChange, - &[ - ("previous", candidate.previous.clone()), - ("current", candidate.current.clone()), - ("renames", renames.into_any()), - ("input_columns", inputs.clone()), - ], + renames_explain_change( + py, + Some(keywords( + py, + &[ + ("previous", candidate.previous.clone()), + ("current", candidate.current.clone()), + ("renames", renames.into_any()), + ("input_columns", inputs.clone()), + ], + )?), + )?, )? .is_truthy()? { @@ -161,20 +168,10 @@ pub(super) fn cursors<'py>( }; targets.set_item(name, target)?; } - let redirected = call_with( - ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, - &[ - ("runtime", phase.runtime.clone()), - ("scope", scope), - ( - "existing_relations", - snapshot.getattr("existing_relations")?, - ), - ("target_relations", targets.into_any()), - ("overrides", phase.overrides.clone()), - ("deferral", phase.deferral.clone()), - ("origin_cursor_columns", origins.into_any()), - ], + let redirected = super::planning_boundary::redirected_cursor_snapshots( + phase, + (snapshot.getattr("existing_relations")?, targets.into_any()), + Some(origins.into_any()), )?; let cursors = imported(py, "builtins", "dict")?.call1((snapshot.getattr("cursor_snapshots")?,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 929aa2a268..66423a9c63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -8,8 +8,6 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -77,21 +75,7 @@ pub(crate) fn plan_column_migrations<'py>( warnings: Vec::new(), }); } - let database = call_with( - ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, - &[("runtime", runtime.clone())], - )?; - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ("adapter", adapter), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ("database", database), - ], - )?; + let state = super::state_inspection::inspection(&runtime, None)?; let schemas = PySet::empty(py)?; for candidate in &candidates { let schema = model_reads::destination(&candidate.model)?.getattr("schema")?; @@ -131,10 +115,7 @@ pub(crate) fn plan_column_migrations<'py>( entries.extend(checked.entries); warnings.extend(checked.warnings); } - let policy = call_with( - ®istry::native(py, NativeAttr::MigrationMissingOriginPolicy)?, - &[("runtime", runtime)], - )?; + let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for entry in &mut entries { *entry = replace(entry, &[("missing_origin_policy", policy.clone())])?; if let Some(warning) = super::column_warnings::entry_warning(entry)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs index fe781f4777..479e79c652 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs @@ -32,7 +32,7 @@ fn projections(shape: &Bound<'_, PyAny>) -> PyResult> { /// Unique automatic column renames whose full query change is explained. #[pyfunction] #[pyo3(signature = (**options))] -fn identical_renames<'py>( +pub(crate) fn identical_renames<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -89,7 +89,7 @@ fn identical_renames<'py>( /// Explicit migration hints for newly added columns resembling removed live columns. #[pyfunction] #[pyo3(signature = (**options))] -fn rename_hints<'py>( +pub(crate) fn rename_hints<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 00f3fa21a4..4a1d4e2c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -2,8 +2,13 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; +use crate::bindings::_helpers::planner_migrations::models::{ + CompatibilityCheck, MigrationCompatibilityResult, +}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -106,30 +111,13 @@ fn migration_snapshot_blocking_findings<'py>( ) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn check_migration_compatibility<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "check_migration_compatibility", - options, - ( - &[ - "model", - "origin_relation", - "origin_columns", - "sql_analysis_enabled", - "column_dialect", - "snapshots_config", - ], - &[], - ), - )?; - let model = arguments.value("model")?; - let relation = arguments.value("origin_relation")?; +/// Whether an origin relation's columns can carry the destination model's history. +pub(super) fn check_migration_compatibility<'py>( + check: &CompatibilityCheck<'py>, +) -> PyResult> { + let py = check.model.py(); + let model = check.model.clone(); + let relation = check.origin_relation.clone(); let relation_type: String = relation.getattr("relation_type")?.extract()?; if relation_type.to_uppercase().contains("VIEW") { return result( @@ -146,34 +134,34 @@ fn check_migration_compatibility<'py>( let columns = PyDict::new(py); columns.set_item( model_reads::name_object(&model)?, - arguments.value("origin_columns")?, + check.origin_columns.clone(), )?; let snapshot = call_with( ®istry::class(py, RecordId::WarehouseSnapshot)?, &[ ("existing_relations", relations.into_any()), ("existing_columns", columns.into_any()), - ("column_dialect", arguments.value("column_dialect")?), + ("column_dialect", check.column_dialect.clone()), ], )?; - let change = call_with( - ®istry::native(py, NativeAttr::DetectModelChanges)?, - &[ - ("model", model.clone()), - ("snapshot", snapshot), - ( - "sql_analysis_enabled", - arguments.value("sql_analysis_enabled")?, - ), - ( - "query_change_tracking", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "full_refresh", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ], + let change = detect_model_changes( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", snapshot), + ("sql_analysis_enabled", check.sql_analysis_enabled.clone()), + ( + "query_change_tracking", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ( + "full_refresh", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ], + )?), )?; let schema = model.getattr("schema_entry")?; let mut has_columns = inferred_rows(&model)?.is_some_and(|rows| !rows.is_empty()); @@ -186,10 +174,10 @@ fn check_migration_compatibility<'py>( return result(py, "not_checked", PyTuple::empty(py)); } let snapshot = config(&model, "materialized")?.as_deref() == Some("snapshot"); - let snapshots_config = arguments.value("snapshots_config")?; + let snapshots_config = check.snapshots_config.clone(); let findings = blocking( (&model, &change.getattr("schema_findings")?), - &arguments.value("column_dialect")?, + &check.column_dialect, snapshot.then_some(&snapshots_config), )?; result( @@ -207,16 +195,14 @@ fn result<'py>( py: Python<'py>, status: &str, findings: Bound<'py, PyTuple>, -) -> PyResult> { - let status = registry::class(py, RecordId::MigrationCompatibility)?.call1((status,))?; - call_with( - ®istry::class(py, RecordId::MigrationCompatibilityResult)?, - &[("status", status), ("findings", findings.into_any())], - ) +) -> PyResult> { + Ok(MigrationCompatibilityResult { + status: registry::class(py, RecordId::MigrationCompatibility)?.call1((status,))?, + findings: findings.into_any(), + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_migration_compatibility, module)?)?; module.add_function(wrap_pyfunction!( migration_incremental_blocking_findings, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index 24535fae12..923c24df27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,61 +1,58 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_migrations::matching::{ + match_migration_fingerprints, migration_destination_fingerprints, +}; +use crate::bindings::_helpers::planner_migrations::models::{ + DiscoveryRequest, FingerprintInputs, ModelMigrationDiscovery, +}; use crate::bindings::_helpers::planner_migrations::{ discovery_events, discovery_models, discovery_requests, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple, PyTupleMethods}; +use std::collections::BTreeSet; -#[pyfunction] -#[pyo3(signature = (**options))] -fn discover_model_migrations<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "discover_model_migrations", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "manual_requests", - "declarations", - "state", - "project_schemas", - "fingerprints", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; - let state = args.value("state")?; - let manual = args.value("manual_requests")?; - let declarations = args.value("declarations")?; +/// Manual and automatically discovered migration requests with their destination fingerprints. +pub(super) fn discover_model_migrations<'py>( + request: DiscoveryRequest<'py>, +) -> PyResult> { + let DiscoveryRequest { + runtime, + scope, + snapshot, + manual_requests: manual, + declarations, + state, + project_schemas, + fingerprints: cache, + } = request; + let py = runtime.py(); let models = discovery_models::ordered(&runtime)?; let declared_names = PyDict::new(py); - for declaration in declarations.try_iter()? { - declared_names.set_item(declaration?.getattr("model_name")?, py.None())?; + for declaration in &declarations { + declared_names.set_item(&declaration.model_name, py.None())?; } - let triggered = - discovery_models::triggered(&models, &scope, &args.value("snapshot")?, &declared_names)?; - if !triggered && !manual.is_truthy()? { - return native(py, RecordId::ModelMigrationDiscovery, &[]); + let triggered = discovery_models::triggered(&models, &scope, &snapshot, &declared_names)?; + if !triggered && manual.is_empty() { + return Ok(ModelMigrationDiscovery { + requests: Vec::new(), + warnings: Vec::new(), + destination_fingerprints: PyDict::new(py), + }); } let metadata = discovery_models::metadata(&runtime, &models)?; - let dialect = runtime - .getattr("adapter")? - .call_method0("sql_analysis_dialect")?; + let inputs = FingerprintInputs { + metadata_jsons: metadata.into_any(), + dialect: runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?, + fingerprints: cache, + }; call_with( &state.getattr("inspect_schemas")?, - &[("schemas", args.value("project_schemas")?)], + &[("schemas", project_schemas)], )?; let declared = discovery_events::declared(&declarations, &state)?; let names = PyDict::new(py); @@ -66,25 +63,15 @@ fn discover_model_migrations<'py>( for (name, origin) in declared.iter() { event_renames.set_item(name, origin)?; } - let seeds = PyTuple::new(py, [event_renames, declared])?; - let cache = args.value("fingerprints")?; + let seeds = PyTuple::new(py, [event_renames, declared])?.into_any(); if !triggered { - let hashes = native( - py, - NativeAttr::MigrationDestinationFingerprints, - &[ - ("requests", manual.clone()), - ("variants", seeds.into_any()), - ("metadata_jsons", metadata.into_any()), - ("dialect", dialect), - ("fingerprints", cache), - ], - )?; - return native( - py, - RecordId::ModelMigrationDiscovery, - &[("requests", manual), ("destination_fingerprints", hashes)], - ); + let destination_fingerprints = + migration_destination_fingerprints(&manual, &seeds, &inputs)?; + return Ok(ModelMigrationDiscovery { + requests: manual, + warnings: Vec::new(), + destination_fingerprints, + }); } let unbuilt = PyDict::new(py); for item in &models { @@ -105,53 +92,26 @@ fn discover_model_migrations<'py>( for (name, _) in resumed.iter() { unbuilt.del_item(name)?; } - let matched = native( - py, - NativeAttr::MatchMigrationFingerprints, - &[ - ("models", PyTuple::new(py, &models)?.into_any()), - ( - "new_names", - PyFrozenSet::new(py, unbuilt.keys().iter())?.into_any(), - ), - ("seed_variants", seeds.into_any()), - ("candidates", candidates.into_any()), - ("metadata_jsons", metadata.clone().into_any()), - ("dialect", dialect.clone()), - ("fingerprints", cache.clone()), - ], - )?; - let mut requests: Vec> = manual.try_iter()?.collect::>()?; + let new_names: BTreeSet = unbuilt + .keys() + .iter() + .map(|name| name.extract()) + .collect::>()?; + let (variant_maps, matches, ambiguous) = + match_migration_fingerprints((&models, new_names), &seeds, candidates.as_any(), &inputs)?; + let mut requests = manual; requests.extend(discovery_requests::automatic( &runtime, &scope, &models, - (&matched.get_item(1)?, &resumed), + (matches.as_any(), &resumed), )?); - let requests = PyTuple::new(py, requests)?; - let warnings = discovery_requests::warnings(&models, &scope, &matched.get_item(2)?)?; - let hashes = native( - py, - NativeAttr::MigrationDestinationFingerprints, - &[ - ("requests", requests.clone().into_any()), - ("variants", matched.get_item(0)?), - ("metadata_jsons", metadata.into_any()), - ("dialect", dialect), - ("fingerprints", cache), - ], - )?; - native( - py, - RecordId::ModelMigrationDiscovery, - &[ - ("requests", requests.into_any()), - ("warnings", warnings.into_any()), - ("destination_fingerprints", hashes), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(discover_model_migrations, module)?) + let warnings = discovery_requests::warnings(&models, &scope, &ambiguous)?; + let destination_fingerprints = + migration_destination_fingerprints(&requests, &variant_maps, &inputs)?; + Ok(ModelMigrationDiscovery { + requests, + warnings: warnings.iter().collect(), + destination_fingerprints, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs index 07254edf57..5b1e8635ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_events.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::config; +use crate::bindings::_helpers::planner_migrations::models::ModelMigrationDeclaration; use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; @@ -32,20 +33,19 @@ pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult( - declarations: &Bound<'py, PyAny>, + declarations: &[ModelMigrationDeclaration<'py>], state: &Bound<'py, PyAny>, ) -> PyResult> { let result = PyDict::new(state.py()); - for item in declarations.try_iter()? { - let item = item?; + for item in declarations { let fingerprint = call_with( &state.getattr("model_fingerprint")?, &[ - ("location", item.getattr("origin_location")?), - ("model_name", item.getattr("origin_model")?), + ("location", item.origin_location.clone()), + ("model_name", item.origin_model.clone()), ], )?; - let explicit = item.getattr("origin_model")?; + let explicit = item.origin_model.clone(); let origin = if explicit.is_truthy()? { explicit } else if !fingerprint.is_none() { @@ -54,7 +54,7 @@ pub(super) fn declared<'py>( state.py().None().into_bound(state.py()) }; if !origin.is_none() { - result.set_item(item.getattr("model_name")?, origin)?; + result.set_item(&item.model_name, origin)?; } } Ok(result) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 41f8d72f48..b7e5b4ad56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -1,13 +1,12 @@ //! Model ordering, selection and metadata at the migration discovery boundary. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_identity::metadata::build_function_local_hashes; +use crate::bindings::_helpers::planner_identity::metadata::build_model_version_identity_metadata_json; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods}; use sqlbuild_analysis::planner_migrations::main::discovery_order::discovery_order; @@ -78,13 +77,15 @@ pub(super) fn metadata<'py>( ) -> PyResult> { let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; - let functions = native( + let functions = object( py, - NativeAttr::BuildFunctionLocalHashes, - &[ - ("functions", project.getattr("functions")?), - ("dialect", project.getattr("sql_analysis_dialect")?), - ], + build_function_local_hashes( + &project.getattr("functions")?, + project + .getattr("sql_analysis_dialect")? + .extract::>()? + .as_deref(), + )?, )?; let identities = host::entry(py, UserCodeHost::BuildHookIdentities)? .call1((project.getattr("hook_functions")?,))?; @@ -100,13 +101,13 @@ pub(super) fn metadata<'py>( for item in models { result.set_item( item.getattr("name")?, - call_with( - ®istry::native(py, NativeAttr::BuildModelVersionIdentityMetadataJson)?, - &[ - ("model", item.clone()), - ("function_local_hashes", functions.clone()), - ("hook_version_hashes", hooks.clone().into_any()), - ], + object( + py, + build_model_version_identity_metadata_json( + item, + functions.extract::>>()?, + hooks.clone().into_any().extract::>>()?, + )?, )?, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index 205e96846b..9f6e729988 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -6,12 +6,14 @@ use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ identity, key, newest, relation, }; +use crate::bindings::_helpers::planner_migrations::models::{ + ModelMigrationDeclaration, ModelMigrationRequest, +}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, + PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PySet, PySetMethods, PyString, PyTuple, }; use sqlbuild_analysis::planner_migrations::main::discovery_materialization::discovery_materialization; use sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint; @@ -48,12 +50,12 @@ pub(super) fn candidates<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, names: &Bound<'py, PyDict>, - claims: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), + claims: (&[ModelMigrationDeclaration<'py>], &Bound<'py, PyDict>), ) -> PyResult> { let py = runtime.py(); let excluded = PySet::empty(py)?; - for declaration in claims.0.try_iter()? { - excluded.add(key(&declaration?.getattr("origin_location")?)?)?; + for declaration in claims.0 { + excluded.add(key(&declaration.origin_location)?)?; } for (_, event) in claims.1.iter() { excluded.add(key(&event.getattr("origin")?)?)?; @@ -107,9 +109,9 @@ pub(super) fn automatic<'py>( scope: &Bound<'py, PyAny>, models: &[Bound<'py, PyAny>], matches: (&Bound<'py, PyAny>, &Bound<'py, PyDict>), -) -> PyResult>> { +) -> PyResult>> { let py = runtime.py(); - let mut requests: Vec> = Vec::new(); + let mut requests: Vec> = Vec::new(); for item in models { if !scope .getattr("selected_keys")? @@ -140,23 +142,16 @@ pub(super) fn automatic<'py>( None }; if let Some((location, origin_model)) = origin { - requests.push(native( - py, - RecordId::ModelMigrationRequest, - &[ - ("model", item.clone()), - ( - "discovery", - registry::class(py, RecordId::MigrationDiscovery)?.getattr("AUTOMATIC")?, - ), - ("origin_location", location), - ("origin_model", origin_model), - ( - "identity_only", - PyBool::new(py, identity(item)?).to_owned().into_any(), - ), - ], - )?); + requests.push(ModelMigrationRequest { + model: item.clone(), + discovery: registry::class(py, RecordId::MigrationDiscovery)? + .getattr("AUTOMATIC")?, + raw_origin: None, + origin_location: Some(location), + origin_model, + force: false, + identity_only: identity(item)?, + }); } } Ok(requests) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 0ce873333a..44eaaf8b95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -12,7 +12,7 @@ use std::collections::BTreeMap; /// Hash normalized SQL and non-storage metadata through rename identities. #[pyfunction] #[pyo3(signature = (**options))] -fn build_migration_fingerprint( +pub(crate) fn build_migration_fingerprint( py: Python<'_>, options: Option>, ) -> PyResult> { @@ -52,7 +52,7 @@ fn build_migration_fingerprint( /// Every name whose rename identity can change the migration fingerprint. #[pyfunction] #[pyo3(signature = (*, query_sql, metadata_json))] -fn migration_fingerprint_ref_names<'py>( +pub(crate) fn migration_fingerprint_ref_names<'py>( py: Python<'py>, query_sql: &str, metadata_json: &str, @@ -73,7 +73,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Attach persisted migration identities through the existing cache boundary. #[pyfunction] #[pyo3(signature = (**options))] -fn with_migration_fingerprints<'py>( +pub(crate) fn with_migration_fingerprints<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index f8106545b7..592daf36bb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,23 +1,25 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_migrations::models::{ + FingerprintInputs, ModelMigrationRequest, +}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::match_migrations::match_migrations; use sqlbuild_analysis::planner_migrations::models::MigrationCandidate; use std::collections::{BTreeMap, BTreeSet}; fn variants<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, + inputs: &FingerprintInputs<'py>, model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { let name = model_reads::name_object(model)?; - let metadata = arguments.value("metadata_jsons")?.get_item(name)?; - let cache = arguments.value("fingerprints")?.getattr("fingerprint")?; + let metadata = inputs.metadata_jsons.get_item(name)?; + let cache = inputs.fingerprints.getattr("fingerprint")?; let mut hashes: Vec = Vec::new(); for map in maps.try_iter()? { let hash: Option = call_with( @@ -26,7 +28,7 @@ fn variants<'py>( ("query_sql", model_reads::query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), - ("dialect", arguments.value("dialect")?), + ("dialect", inputs.dialect.clone()), ], )? .extract()?; @@ -36,105 +38,62 @@ fn variants<'py>( hashes.push(hash); } } - let _ = py; Ok(hashes) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn match_migration_fingerprints<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "match_migration_fingerprints", - options, - ( - &[ - "models", - "new_names", - "seed_variants", - "candidates", - "metadata_jsons", - "dialect", - "fingerprints", - ], - &[], - ), - )?; - let new_names: BTreeSet = arguments.value("new_names")?.extract()?; - let mut models: BTreeMap> = BTreeMap::new(); +/// The fingerprint variants tried, the matched candidate per new model, and ambiguous names. +pub(super) fn match_migration_fingerprints<'py>( + (models, new_names): (&[Bound<'py, PyAny>], BTreeSet), + seed_variants: &Bound<'py, PyAny>, + candidates: &Bound<'py, PyAny>, + inputs: &FingerprintInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyAny>)> { + let py = seed_variants.py(); + let mut by_name: BTreeMap> = BTreeMap::new(); let mut names: Vec = Vec::new(); - for model in arguments.value("models")?.try_iter()? { - let model = model?; - let name: String = model_reads::model_name(&model)?; + for model in models { + let name: String = model_reads::model_name(model)?; if new_names.contains(&name) { names.push(name.clone()); } - models.insert(name, model); + by_name.insert(name, model.clone()); } - let original: Vec> = arguments - .value("candidates")? - .try_iter()? - .collect::>()?; - let mut candidates: Vec = Vec::new(); + let original: Vec> = candidates.try_iter()?.collect::>()?; + let mut fingerprints: Vec = Vec::new(); for candidate in &original { - candidates.push(MigrationCandidate { + fingerprints.push(MigrationCandidate { name: candidate.getattr("node_name")?.extract()?, fingerprint: stored_migration_fingerprint(candidate)?.unwrap_or_default(), }); } - let seeds: Vec> = arguments.value("seed_variants")?.extract()?; - let matched = match_migrations(&names, &seeds, &candidates, |name, maps| { - variants( - py, - &arguments, - &models[name], - PyTuple::new(py, maps)?.into_any(), - ) + let seeds: Vec> = seed_variants.extract()?; + let matched = match_migrations(&names, &seeds, &fingerprints, |name, maps| { + variants(inputs, &by_name[name], PyTuple::new(py, maps)?.into_any()) })?; let matches = PyDict::new(py); for (name, index) in matched.matches { matches.set_item(name, &original[index])?; } - PyTuple::new( - py, - [ - PyTuple::new(py, matched.variants)?.into_any(), - matches.into_any(), - matched.ambiguous.into_pyobject(py)?.into_any(), - ], - ) + Ok(( + PyTuple::new(py, matched.variants)?.into_any(), + matches, + matched.ambiguous.into_pyobject(py)?.into_any(), + )) } -#[pyfunction] -#[pyo3(signature = (**options))] -fn migration_destination_fingerprints<'py>( - py: Python<'py>, - options: Option>, +/// The fingerprint variants of each requested destination model with metadata. +pub(super) fn migration_destination_fingerprints<'py>( + requests: &[ModelMigrationRequest<'py>], + variant_maps: &Bound<'py, PyAny>, + inputs: &FingerprintInputs<'py>, ) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "migration_destination_fingerprints", - options, - ( - &[ - "requests", - "variants", - "metadata_jsons", - "dialect", - "fingerprints", - ], - &[], - ), - )?; + let py = variant_maps.py(); let result = PyDict::new(py); - for request in arguments.value("requests")?.try_iter()? { - let model = request?.getattr("model")?; - let name = model_reads::name_object(&model)?; - if arguments - .value("metadata_jsons")? + for request in requests { + let model = &request.model; + let name = model_reads::name_object(model)?; + if inputs + .metadata_jsons .call_method1("get", (&name,))? .is_none() { @@ -142,26 +101,16 @@ fn migration_destination_fingerprints<'py>( } result.set_item( name, - PyTuple::new( - py, - variants(py, &arguments, &model, arguments.value("variants")?)?, - )?, + PyTuple::new(py, variants(inputs, model, variant_maps.clone())?)?, )?; } Ok(result) } -#[pyfunction] -fn stored_migration_fingerprint(fingerprint: &Bound<'_, PyAny>) -> PyResult> { +/// The migration fingerprint stored in a model fingerprint's metadata. +pub(super) fn stored_migration_fingerprint( + fingerprint: &Bound<'_, PyAny>, +) -> PyResult> { let metadata: String = fingerprint.getattr("metadata_json")?.extract()?; Ok(sqlbuild_analysis::planner_migrations::main::stored_migration_fingerprint::stored_migration_fingerprint(&metadata)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(match_migration_fingerprints, module)?)?; - module.add_function(wrap_pyfunction!( - migration_destination_fingerprints, - module - )?)?; - module.add_function(wrap_pyfunction!(stored_migration_fingerprint, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index b59d161658..9b2da1d432 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -12,7 +12,7 @@ mod discovery_events; mod discovery_models; mod discovery_requests; mod errors; -mod fingerprint; +pub(crate) mod fingerprint; mod known_input_columns; mod matching; mod model_entries; @@ -25,10 +25,10 @@ mod old_name_entries; mod old_name_objects; pub(crate) mod old_name_planning; mod old_name_references; -mod old_names; +pub(crate) mod old_names; mod planning_boundary; mod planning_execution; mod planning_origin; -mod projections; +pub(crate) mod projections; pub(crate) mod registration; mod state_inspection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index 59a4408c60..aa28283afe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -1,19 +1,21 @@ //! Assemble model migration decisions from inspected warehouse facts. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_migrations::column_objects::{ - lower, model, native, replace, text, -}; +use crate::bindings::_helpers::planner_migrations::column_objects::{lower, model, replace, text}; +use crate::bindings::_helpers::planner_migrations::compatibility::check_migration_compatibility; use crate::bindings::_helpers::planner_migrations::model_handover::{ applies, empty_version, handover, }; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_migrations::models::{ + CompatibilityCheck, MigrationDecisionRequest, +}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_move; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyBool, PyTuple}; +use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; pub(super) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { @@ -44,30 +46,35 @@ pub(super) fn decision<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult( entry: Bound<'py, PyAny>, - fingerprint: Bound<'py, PyAny>, + inherited: Bound<'py, PyAny>, applies: bool, -) -> PyResult> { - let py = entry.py(); - Ok(PyTuple::new( - py, - [ - entry, - fingerprint, - PyBool::new(py, applies).to_owned().into_any(), - ], - )? - .into_any()) +) -> PyResult> { + Ok(MigrationDecision { + entry, + inherited, + applies, + }) +} + +/// The decided plan entry, the inherited fingerprint, and whether the history handover applies. +pub(super) struct MigrationDecision<'py> { + pub(super) entry: Bound<'py, PyAny>, + pub(super) inherited: Bound<'py, PyAny>, + pub(super) applies: bool, } -pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult> { - let runtime = args.value("runtime")?; +pub(super) fn decide<'py>( + decision_request: &MigrationDecisionRequest<'_, 'py>, +) -> PyResult> { + let runtime = decision_request.runtime.clone(); let py = runtime.py(); - let request = args.value("request")?; - let item = request.getattr("model")?; + let request = decision_request.request; + let item = request.model.clone(); let destination = item.getattr("destination")?; - let origin = args.value("origin")?; - let snapshot = args.value("snapshot")?; - let state = args.value("state")?; + let origin = decision_request.origin.clone(); + let snapshot = decision_request.snapshot.clone(); + let state = decision_request.state.clone(); + let origin_model = decision_request.origin_model.clone(); if key(&origin)? == key(&destination)? { let name: String = item.getattr("name")?.extract()?; return Err(pyo3::PyErr::from_value( @@ -82,7 +89,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult(args: &KeywordArguments<'py>) -> PyResult( ) -> PyResult> { let py = model.py(); if handover.is_none() - || !destinations.contains( - registry::native(py, NativeAttr::StoredMigrationFingerprint)?.call1((&handover,))?, - )? + || !destinations.contains(super::matching::stored_migration_fingerprint(&handover)?)? { return Ok(handover); } @@ -93,13 +91,13 @@ pub(super) fn equivalent<'py>( ("definition", model_reads::query_sql(model)?), ( "definition_hash", - native( + object( py, - NativeAttr::ComputeQueryHash, - &[ - ("query_sql", model_reads::query_sql(model)?), - ("dialect", dialect.clone()), - ], + compute_query_hash( + py, + &model_reads::query_sql(model)?.extract::()?, + dialect.extract::>()?.as_deref(), + )?, )?, ), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index 6d8cca941a..1b9941282b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -1,12 +1,10 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -89,16 +87,10 @@ pub(super) fn overlay<'py>( .getattr("cursor_snapshots")? .cast_into::()? .copy()?; - let gathered = call_with( - ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, - &[ - ("runtime", runtime), - ("scope", phase.scope.clone()), - ("existing_relations", relations.clone().into_any()), - ("target_relations", redirected.into_any()), - ("overrides", phase.overrides.clone()), - ("deferral", phase.deferral.clone()), - ], + let gathered = super::planning_boundary::redirected_cursor_snapshots( + phase, + (relations.clone().into_any(), redirected.into_any()), + None, )?; cursors.call_method1("update", (gathered,))?; replace( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 515c7577c7..093f5c8be9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -1,20 +1,27 @@ //! Model migration orchestration over Python-owned live inspection. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_changes::renames::plan_reference_renames; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; +use crate::bindings::_helpers::planner_migrations::discovery::discover_model_migrations; use crate::bindings::_helpers::planner_migrations::models::{ - MigrationPhase, ModelMigrationPlanning, + DiscoveryRequest, MigrationDecisionRequest, MigrationPhase, ModelMigrationDeclaration, + ModelMigrationDiscovery, ModelMigrationPlanning, ModelMigrationRequest, }; +use crate::bindings::_helpers::planner_migrations::planning_origin::{ + require_same_database, resolve_origin, +}; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyString, PyTuple, }; -use pyo3::{pyfunction, wrap_pyfunction}; -fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult>> { +fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult>> { let py = item.py(); let values = item.getattr("config")?.getattr("values")?; let origin = values.call_method1("get", ("migrate_from",))?; @@ -22,44 +29,23 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult> return Ok(None); } let materialization = config(item, "materialized")?; - Ok(Some(native( - py, - RecordId::ModelMigrationRequest, - &[ - ("model", item.clone()), - ( - "discovery", - registry::class(py, RecordId::MigrationDiscovery)?.getattr("MANUAL")?, - ), - ("raw_origin", origin.call_method0("strip")?), - ( - "force", - PyBool::new( - py, - values - .call_method1("get", ("migrate_force",))? - .is(PyBool::new(py, true)), - ) - .to_owned() - .into_any(), - ), - ( - "identity_only", - PyBool::new( - py, - materialization.eq("table")? || materialization.eq("view")?, - ) - .to_owned() - .into_any(), - ), - ], - )?)) + Ok(Some(ModelMigrationRequest { + model: item.clone(), + discovery: registry::class(py, RecordId::MigrationDiscovery)?.getattr("MANUAL")?, + raw_origin: Some(origin.call_method0("strip")?), + origin_location: None, + origin_model: py.None().into_bound(py), + force: values + .call_method1("get", ("migrate_force",))? + .is(PyBool::new(py, true)), + identity_only: materialization.eq("table")? || materialization.eq("view")?, + })) } -#[pyfunction] -#[pyo3(signature = (*, scope))] -fn manual_migration_requests<'py>(scope: &Bound<'py, PyAny>) -> PyResult> { - let mut requests: Vec> = Vec::new(); +fn manual_migration_requests<'py>( + scope: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut requests: Vec> = Vec::new(); for key in scope.getattr("execution_order")?.try_iter()? { let key = key?; if !scope.getattr("selected_keys")?.contains(&key)? @@ -76,32 +62,7 @@ fn manual_migration_requests<'py>(scope: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - options: Option>, -) -> PyResult> { - super::model_entries::decide(&KeywordArguments::read( - py, - "decide_model_migration", - options, - ( - &[ - "runtime", - "request", - "origin", - "origin_model", - "snapshot", - "state", - "snapshots_config", - ], - &[], - ), - )?) + Ok(requests) } /// Model migrations of the selected scope: discovered and declared moves, decided against live state. @@ -109,34 +70,11 @@ pub(crate) fn plan_model_migrations<'py>( phase: &MigrationPhase<'py>, fingerprints: &Bound<'py, PyAny>, ) -> PyResult> { - let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); - let schemas = call_with( - ®istry::native(py, NativeAttr::MigrationProjectSchemas)?, - &[("runtime", runtime.clone())], - )?; - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ( - "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ( - "database", - call_with( - ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, - &[("runtime", runtime.clone())], - )?, - ), - ("known_fingerprints", snapshot.getattr("fingerprints")?), - ], - )?; + let schemas = super::planning_origin::project_schemas(&runtime)?; + let state = + super::state_inspection::inspection(&runtime, Some(&snapshot.getattr("fingerprints")?))?; let discovery = discover((phase, fingerprints), &state, &schemas)?; finish(phase, &state, &schemas, &discovery) } @@ -145,12 +83,11 @@ fn discover<'py>( (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let runtime = phase.runtime.clone(); - let py = runtime.py(); let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); - let mut declared_requests: Vec> = Vec::new(); + let mut declared_requests: Vec> = Vec::new(); for item in runtime_field(&runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? @@ -165,63 +102,45 @@ fn discover<'py>( &[("schemas", schemas.clone())], )?; } - let mut declarations: Vec> = Vec::new(); - for request in declared_requests { - let resolved = call_with( - ®istry::native(py, NativeAttr::MigrationResolveOrigin)?, - &[ - ("request", request.clone()), - ("runtime", runtime.clone()), - ("state", state.clone()), - ], - )?; - declarations.push(native( - py, - RecordId::ModelMigrationDeclaration, - &[ - ("model_name", request.getattr("model")?.getattr("name")?), - ("origin_location", resolved.get_item(0)?), - ("origin_model", resolved.get_item(1)?), - ], - )?); + let mut declarations: Vec> = Vec::new(); + for request in &declared_requests { + let (origin_location, origin_model) = resolve_origin(request, &runtime, state)?; + declarations.push(ModelMigrationDeclaration { + model_name: request.model.getattr("name")?, + origin_location, + origin_model, + }); } - call_with( - ®istry::native(py, NativeAttr::DiscoverModelMigrations)?, - &[ - ("runtime", runtime.clone()), - ("scope", scope.clone()), - ("snapshot", snapshot.clone()), - ("manual_requests", manual_migration_requests(&scope)?), - ("declarations", PyTuple::new(py, declarations)?.into_any()), - ("state", state.clone()), - ("project_schemas", schemas.clone()), - ("fingerprints", fingerprints.clone()), - ], - ) + discover_model_migrations(DiscoveryRequest { + manual_requests: manual_migration_requests(&scope)?, + runtime, + scope, + snapshot, + declarations, + state: state.clone(), + project_schemas: schemas.clone(), + fingerprints: fingerprints.clone(), + }) } fn finish<'py>( phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, - discovery: &Bound<'py, PyAny>, + discovery: &ModelMigrationDiscovery<'py>, ) -> PyResult> { let runtime = phase.runtime.clone(); let py = runtime.py(); let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); - let mut resolved: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for request in discovery.getattr("requests")?.try_iter()? { - let request = request?; - let origin = call_with( - ®istry::native(py, NativeAttr::MigrationResolveOrigin)?, - &[ - ("request", request.clone()), - ("runtime", runtime.clone()), - ("state", state.clone()), - ], - )?; - resolved.push((request, origin.get_item(0)?, origin.get_item(1)?)); + let mut resolved: Vec<( + &ModelMigrationRequest<'py>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + )> = Vec::new(); + for request in &discovery.requests { + let (origin, origin_model) = resolve_origin(request, &runtime, state)?; + resolved.push((request, origin, origin_model)); } let mut entries: Vec> = Vec::new(); let handovers = PyDict::new(py); @@ -229,10 +148,7 @@ fn finish<'py>( let origin_schemas = PySet::empty(py)?; let mut origins: Vec> = Vec::new(); for (_, origin, _) in &resolved { - call_with( - ®istry::native(py, NativeAttr::MigrationRequireSameDatabase)?, - &[("state", state.clone()), ("location", origin.clone())], - )?; + require_same_database(state, origin)?; let schema = origin.getattr("schema")?; if !schema.is_none() { origin_schemas.add(schema)?; @@ -253,39 +169,32 @@ fn finish<'py>( } else { project_config.getattr("snapshots")? }; - let policy = call_with( - ®istry::native(py, NativeAttr::MigrationMissingOriginPolicy)?, - &[("runtime", runtime.clone())], - )?; + let policy = super::planning_boundary::missing_origin_policy(&runtime)?; for (request, origin, origin_model) in resolved { - let item = request.getattr("model")?; + let item = request.model.clone(); let name = item.getattr("name")?; - let decided = native( - py, - NativeAttr::DecideModelMigration, - &[ - ("runtime", runtime.clone()), - ("request", request), - ("origin", origin), - ("origin_model", origin_model), - ("snapshot", snapshot.clone()), - ("state", state.clone()), - ("snapshots_config", snapshots_config.clone()), - ], - )?; + let decided = super::model_entries::decide(&MigrationDecisionRequest { + runtime: runtime.clone(), + request, + origin, + origin_model, + snapshot: snapshot.clone(), + state: state.clone(), + snapshots_config: snapshots_config.clone(), + })?; entries.push(replace( - &decided.get_item(0)?, + &decided.entry, &[("missing_origin_policy", policy.clone())], )?); - if decided.get_item(2)?.is_truthy()? { + if decided.applies { let destinations = discovery - .getattr("destination_fingerprints")? + .destination_fingerprints .call_method1("get", (&name, PyTuple::empty(py)))?; handovers.set_item( name, super::model_handover::equivalent( &item, - decided.get_item(1)?, + decided.inherited, &destinations, &snapshot.getattr("column_dialect")?, )?, @@ -296,9 +205,7 @@ fn finish<'py>( let planned = PyDict::new(py); let mut renamed: Vec> = Vec::new(); let mut warnings: Vec> = Vec::new(); - for warning in discovery.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } + warnings.extend(discovery.warnings.iter().cloned()); for entry in &entries { let name = entry.getattr("model_name")?; if let Some(handover) = handovers.get_item(&name)? { @@ -319,17 +226,22 @@ fn finish<'py>( } else { super::model_overlay::overlay(phase, state, &entries, &handovers)? }; - let reference_renames = native( + let reference_renames = object( py, - NativeAttr::PlanReferenceRenames, - &[ - ("runtime", runtime), - ("scope", scope), - ("snapshot", snapshot), - ("state", state.clone()), - ("schemas", schemas.clone()), - ("planned", planned.into_any()), - ], + plan_reference_renames( + py, + Some(keywords( + py, + &[ + ("runtime", runtime), + ("scope", scope), + ("snapshot", snapshot), + ("state", state.clone()), + ("schemas", schemas.clone()), + ("planned", planned.into_any()), + ], + )?), + )?, )?; let mut fields = vec![("reference_renames", reference_renames)]; if !entries.is_empty() { @@ -341,8 +253,3 @@ fn finish<'py>( warnings, }) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(manual_migration_requests, module)?)?; - module.add_function(wrap_pyfunction!(decide_model_migration, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index abac12f8cd..4d04495439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -69,3 +69,82 @@ pub(crate) struct OldNameViewPlanning<'py> { pub(crate) views: Bound<'py, PyAny>, pub(crate) warnings: Vec>, } + +/// One destination model whose data should come from an earlier relation. +#[derive(Debug, Clone)] +pub(crate) struct ModelMigrationRequest<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) discovery: Bound<'py, PyAny>, + pub(crate) raw_origin: Option>, + pub(crate) origin_location: Option>, + pub(crate) origin_model: Bound<'py, PyAny>, + pub(crate) force: bool, + pub(crate) identity_only: bool, +} + +/// One project model's explicit migrate_from, resolved to a relation. +#[derive(Debug)] +pub(crate) struct ModelMigrationDeclaration<'py> { + pub(crate) model_name: Bound<'py, PyAny>, + pub(crate) origin_location: Bound<'py, PyAny>, + pub(crate) origin_model: Bound<'py, PyAny>, +} + +/// Manual and automatically discovered migration requests for one plan. +#[derive(Debug)] +pub(crate) struct ModelMigrationDiscovery<'py> { + pub(crate) requests: Vec>, + pub(crate) warnings: Vec>, + pub(crate) destination_fingerprints: Bound<'py, PyDict>, +} + +/// Compatibility status plus human-readable blocking findings. +#[derive(Debug)] +pub(crate) struct MigrationCompatibilityResult<'py> { + pub(crate) status: Bound<'py, PyAny>, + pub(crate) findings: Bound<'py, PyAny>, +} + +/// The fingerprint inputs shared by migration matching and destination fingerprints. +#[derive(Debug)] +pub(crate) struct FingerprintInputs<'py> { + pub(crate) metadata_jsons: Bound<'py, PyAny>, + pub(crate) dialect: Bound<'py, PyAny>, + pub(crate) fingerprints: Bound<'py, PyAny>, +} + +/// One resolved migration request decided against live state. +#[derive(Debug)] +pub(crate) struct MigrationDecisionRequest<'a, 'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) request: &'a ModelMigrationRequest<'py>, + pub(crate) origin: Bound<'py, PyAny>, + pub(crate) origin_model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) state: Bound<'py, PyAny>, + pub(crate) snapshots_config: Bound<'py, PyAny>, +} + +/// What checking an origin relation's compatibility with its destination model reads. +#[derive(Debug)] +pub(crate) struct CompatibilityCheck<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) origin_relation: Bound<'py, PyAny>, + pub(crate) origin_columns: Bound<'py, PyAny>, + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) column_dialect: Bound<'py, PyAny>, + pub(crate) snapshots_config: Bound<'py, PyAny>, +} + +/// What discovering automatic and manual migration requests reads. +#[derive(Debug)] +pub(crate) struct DiscoveryRequest<'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) manual_requests: Vec>, + pub(crate) declarations: Vec>, + pub(crate) state: Bound<'py, PyAny>, + pub(crate) project_schemas: Bound<'py, PyAny>, + pub(crate) fingerprints: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 0b3fb70282..6d9a45f2e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,14 +1,18 @@ //! Assemble new and resumed old-name view steps from recorded facts. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; use crate::bindings::_helpers::planner_migrations::old_name_objects::{ expires, location, relation, retention, stores_history, }; +use crate::bindings::_helpers::planner_migrations::old_names::declared_column_aliases; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_view_skip_reason; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; @@ -136,26 +140,31 @@ impl<'py> OldNamePlanner<'py> { tracked: Bound<'py, PyAny>, retention: &Bound<'py, PyAny>, ) -> PyResult> { - native( + object( self.runtime.py(), - NativeAttr::OldNameViewSkipReason, - &[ - ("origin", origin.clone()), - ("origin_tracked", tracked), - ("retention", retention.clone()), - ( - "models", - PyTuple::new(self.runtime.py(), &self.models)?.into_any(), - ), - ("views", self.views.clone()), - ], + old_name_view_skip_reason( + self.runtime.py(), + Some(keywords( + self.runtime.py(), + &[ + ("origin", origin.clone()), + ("origin_tracked", tracked), + ("retention", retention.clone()), + ( + "models", + PyTuple::new(self.runtime.py(), &self.models)?.into_any(), + ), + ("views", self.views.clone()), + ], + )?), + )?, ) } fn aliases(&self, item: &Bound<'py, PyAny>) -> PyResult> { let py = self.runtime.py(); if !stores_history(item)? { - return registry::native(py, NativeAttr::DeclaredColumnAliases)?.call1((item,)); + return Ok(declared_column_aliases(py, item)?.into_any()); } let mut aliases: Vec> = Vec::new(); for entry in self.columns.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index ff0045eecc..10a26869ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -1,14 +1,15 @@ //! Old-name planning orchestration over existing live state inspection. use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_claim_errors; +use crate::bindings::_helpers::planner_migrations::old_names::old_name_schemas_to_read; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; @@ -52,19 +53,14 @@ fn schemas<'py>( } } if include_project { - let project = native( - py, - NativeAttr::MigrationProjectSchemas, - &[("runtime", runtime)], - )?; + let project = super::planning_origin::project_schemas(&runtime)?; for schema in project.try_iter()? { needed.add(schema?)?; } } - native( + object( py, - NativeAttr::OldNameSchemasToRead, - &[("snapshot", snapshot), ("needed", needed.into_any())], + old_name_schemas_to_read(py, &snapshot, &needed.into_any())?, ) } @@ -95,27 +91,7 @@ pub(crate) fn plan_old_name_views<'py>( selected.push(item); } } - let state = call_with( - ®istry::native(py, NativeAttr::MigrationStateInspection)?, - &[ - ( - "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ( - "database", - native( - py, - NativeAttr::MigrationPlanningDatabase, - &[("runtime", runtime.clone())], - )?, - ), - ], - )?; + let state = super::state_inspection::inspection(&runtime, None)?; let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( @@ -164,18 +140,23 @@ pub(crate) fn plan_old_name_views<'py>( } } let entries = PyTuple::new(py, entries)?.into_any(); - let claims = native( + let claims = object( py, - NativeAttr::OldNameClaimErrors, - &[ - ("selected", PyTuple::new(py, selected)?.into_any()), - ("snapshot", phase.snapshot.clone()), - ("views", views.clone()), - ( - "now", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, - ), - ], + old_name_claim_errors( + py, + Some(keywords( + py, + &[ + ("selected", PyTuple::new(py, selected)?.into_any()), + ("snapshot", phase.snapshot.clone()), + ("views", views.clone()), + ( + "now", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), + ], + )?), + )?, )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; warnings.extend(super::old_name_references::errors( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index 994f90033c..763428c7d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -1,11 +1,15 @@ //! Diagnose literal references to compatibility-only old names. +use crate::bindings::_helpers::analysis::explicit_references::hard_coded_relation_remedy; +use crate::bindings::_helpers::analysis::explicit_references::old_name_reference_message; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::relations::match_project_relation; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; @@ -83,11 +87,7 @@ pub(super) fn errors<'py>( for literal in literals.try_iter()? { let literal = literal?; let relation = literal.getattr("relation")?; - let matched = native( - py, - NativeAttr::MatchProjectRelation, - &[("index", index.clone()), ("relation", relation.clone())], - )?; + let matched = object(py, match_project_relation(&index, &relation, None, None)?)?; if matched.is_none() || matched.getattr("compatibility_for")?.is_none() { continue; } @@ -98,26 +98,21 @@ pub(super) fn errors<'py>( parts.push(part.extract()?); } } - let message: String = native( + let message: String = object( py, - NativeAttr::OldNameReferenceMessage, - &[ - ("owner_label", literal.getattr("owner_label")?), - ("written", text(py, &parts.join("."))), - ("method", literal.getattr("method")?), - ("model_name", matched.getattr("compatibility_for")?), - ], - )? - .extract()?; - let remedy: String = native( - py, - NativeAttr::HardCodedRelationRemedy, - &[ - ("owner_kind", literal.getattr("owner_kind")?), - ("ref", matched.getattr("ref")?), - ], + old_name_reference_message( + &literal.getattr("owner_label")?.extract::()?, + &parts.join("."), + &literal.getattr("method")?.extract::()?, + &matched.getattr("compatibility_for")?.extract::()?, + ), )? .extract()?; + let remedy: String = hard_coded_relation_remedy( + &literal.getattr("owner_kind")?.extract::()?, + &matched.getattr("ref")?, + None, + )?; let path = literal .getattr("relative_path")? .str()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 8ebfdd6972..b6a5675b08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -16,7 +16,7 @@ use sqlbuild_analysis::planner_migrations::main::{ }; #[pyfunction] -fn declared_column_aliases<'py>( +pub(crate) fn declared_column_aliases<'py>( py: Python<'py>, model: &Bound<'py, PyAny>, ) -> PyResult> { @@ -36,7 +36,7 @@ fn declared_column_aliases<'py>( #[pyfunction] #[pyo3(signature = (*, snapshot, needed))] -fn old_name_schemas_to_read<'py>( +pub(crate) fn old_name_schemas_to_read<'py>( py: Python<'py>, snapshot: &Bound<'py, PyAny>, needed: &Bound<'py, PyAny>, @@ -97,7 +97,7 @@ fn claim_message(model: &str, view: &Bound<'_, PyAny>, now: &Bound<'_, PyAny>) - #[pyfunction] #[pyo3(signature = (**options))] -fn old_name_claim_errors<'py>( +pub(crate) fn old_name_claim_errors<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -155,7 +155,7 @@ fn old_name_claim_errors<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn old_name_view_skip_reason<'py>( +pub(crate) fn old_name_view_skip_reason<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index 7afa215fba..97951f109f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -1,53 +1,34 @@ //! Assemble migration policy and redirected live cursor inspection requests. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::retention::effective_target_config; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::planner_warehouse::snapshot::gather_redirected_cursor_snapshots; +use crate::bindings::_helpers::records::ids::RecordId; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -#[pyfunction] -#[pyo3(signature = (*, runtime))] -fn migration_missing_origin_policy<'py>( +/// The target's policy for migration origins that no longer exist. +pub(super) fn missing_origin_policy<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - ®istry::native(runtime.py(), NativeAttr::EffectiveTargetConfig)?, - &[("runtime", runtime.clone())], - )? - .getattr("missing_migration_origin") + effective_target_config(runtime)?.getattr("missing_migration_origin") } -#[pyfunction] -#[pyo3(signature = (**options))] -fn migration_redirected_cursor_snapshots<'py>( - py: Python<'py>, - options: Option>, +/// Live cursor snapshots of models whose history now sits in a redirected relation. +pub(super) fn redirected_cursor_snapshots<'py>( + phase: &MigrationPhase<'py>, + (existing_relations, target_relations): (Bound<'py, PyAny>, Bound<'py, PyAny>), + origin_cursor_columns: Option>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "migration_redirected_cursor_snapshots", - options, - ( - &[ - "runtime", - "scope", - "existing_relations", - "target_relations", - "overrides", - "deferral", - ], - &["origin_cursor_columns"], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; - let overrides = args.value("overrides")?; - let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; + let runtime = &phase.runtime; + let py = runtime.py(); + let overrides = &phase.overrides; + let scope = &phase.scope; + let project = runtime_field(runtime, |runtime| &runtime.project)?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let start = if config.is_none() { py.None().into_bound(py) } else { @@ -61,55 +42,47 @@ fn migration_redirected_cursor_snapshots<'py>( ("runtime_producer_keys", scope.getattr("selected_keys")?), ( "invocation_time", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + runtime_field(runtime, |runtime| &runtime.invocation_time)?, ), ("start_cursor_config", start), ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], )?; - let full_refresh = call_with( - ®istry::native(py, NativeAttr::EffectivelyFullRefreshedModelNames)?, - &[ - ("project", project.clone()), - ("cli_full_refresh", overrides.getattr("full_refresh")?), - ], + let full_refresh = effectively_full_refreshed_model_names( + &project, + overrides.getattr("full_refresh")?.is_truthy()?, )?; - call_with( - ®istry::native(py, NativeAttr::GatherRedirectedCursorSnapshots)?, - &[ - ("project", project), - ( - "adapter", - runtime_field(&runtime, |runtime| &runtime.adapter)?, - ), - ( - "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, - ), - ("existing_relations", args.value("existing_relations")?), - ("target_relations", args.value("target_relations")?), - ("cursor_scope", cursor_scope), - ("full_refresh_model_names", full_refresh), - ( - "deferred_locations", - args.value("deferral")?.getattr("deferred_locations")?, - ), - ( - "on_progress", - runtime_field(&runtime, |runtime| &runtime.on_progress)?, - ), - ( - "origin_cursor_columns", - args.value("origin_cursor_columns")?, - ), - ], + gather_redirected_cursor_snapshots( + py, + Some(keywords( + py, + &[ + ("project", project), + ( + "adapter", + runtime_field(runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(runtime, |runtime| &runtime.connection)?, + ), + ("existing_relations", existing_relations), + ("target_relations", target_relations), + ("cursor_scope", cursor_scope), + ("full_refresh_model_names", full_refresh), + ( + "deferred_locations", + phase.deferral.getattr("deferred_locations")?, + ), + ( + "on_progress", + runtime_field(runtime, |runtime| &runtime.on_progress)?, + ), + ( + "origin_cursor_columns", + origin_cursor_columns.unwrap_or_else(|| py.None().into_bound(py)), + ), + ], + )?), ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_missing_origin_policy, module)?)?; - module.add_function(wrap_pyfunction!( - migration_redirected_cursor_snapshots, - module - )?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs index 2fd4af82e1..97a010b63c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -7,9 +7,8 @@ use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; -use pyo3::{pyfunction, wrap_pyfunction}; fn label<'py>(location: &Bound<'py, PyAny>) -> PyResult> { let qualified = location.getattr("qualified_name")?; @@ -20,8 +19,6 @@ fn label<'py>(location: &Bound<'py, PyAny>) -> PyResult> { } } -#[pyfunction] -#[pyo3(signature = (*, runtime, decision))] fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, @@ -40,9 +37,8 @@ fn migration_promotion<'py>( promotion.getattr("RENAME") } -#[pyfunction] -#[pyo3(signature = (*, runtime, model, entry))] -fn migration_with_execution<'py>( +/// The migration entry with the adapter's staging, transfer and promotion choices. +pub(super) fn migration_with_execution<'py>( runtime: &Bound<'py, PyAny>, model: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, @@ -91,8 +87,3 @@ fn migration_with_execution<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_promotion, module)?)?; - module.add_function(wrap_pyfunction!(migration_with_execution, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 0f3289d8a2..de08a94111 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -2,12 +2,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::bindings::_helpers::planner_migrations::models::ModelMigrationRequest; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PySet, PySetMethods, PyTuple}; -use pyo3::{PyErr, pyfunction, wrap_pyfunction}; +use pyo3::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PySet, PySetMethods}; pub(super) fn location<'py>( runtime: &Bound<'py, PyAny>, @@ -32,17 +33,14 @@ fn named<'py>( destination: &Bound<'py, PyAny>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { let item = item?; if item.getattr("name")?.eq(name)? { - return Ok( - PyTuple::new(runtime.py(), [item.getattr("destination")?, name.clone()])? - .into_any(), - ); + return Ok((item.getattr("destination")?, name.clone())); } } let fingerprint = call_with( @@ -62,7 +60,7 @@ fn named<'py>( ("name", name.clone()), ] }; - Ok(PyTuple::new(runtime.py(), [location(runtime, &fields)?, name.clone()])?.into_any()) + Ok((location(runtime, &fields)?, name.clone())) } fn input_error(py: Python<'_>, message: String) -> PyResult { @@ -71,9 +69,8 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { )) } -#[pyfunction] -#[pyo3(signature = (*, runtime))] -fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { +/// The database migration planning inspects: the first model destination's, else the target's. +pub(super) fn planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = project.getattr("effective_target_database")?; for item in project.getattr("models")?.try_iter()? { @@ -85,9 +82,8 @@ fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { +/// The schemas the project's models write to. +pub(super) fn project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? @@ -101,9 +97,8 @@ fn migration_project_schemas<'py>(runtime: &Bound<'py, PyAny>) -> PyResult, location: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -133,25 +128,21 @@ fn migration_require_same_database( )?) } -#[pyfunction] -#[pyo3(signature = (*, request, runtime, state))] -fn migration_resolve_origin<'py>( - request: &Bound<'py, PyAny>, +/// The origin location and origin model name of one migration request. +pub(super) fn resolve_origin<'py>( + request: &ModelMigrationRequest<'py>, runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = runtime.py(); - let explicit = request.getattr("origin_location")?; - if !explicit.is_none() { - return Ok(PyTuple::new(py, [explicit, request.getattr("origin_model")?])?.into_any()); + if let Some(explicit) = &request.origin_location { + return Ok((explicit.clone(), request.origin_model.clone())); } - let raw = request.getattr("raw_origin")?; - let raw = if raw.is_truthy()? { - raw - } else { - super::column_objects::text(py, "") + let raw = match &request.raw_origin { + Some(raw) if raw.is_truthy()? => raw.clone(), + _ => super::column_objects::text(py, ""), }; - let item = request.getattr("model")?; + let item = request.model.clone(); let destination = item.getattr("destination")?; let mut parts: Vec> = Vec::new(); for part in raw.call_method1("split", (".",))?.try_iter()? { @@ -191,12 +182,5 @@ fn migration_resolve_origin<'py>( ("name", parts[parts.len() - 1].clone()), ], )?; - Ok(PyTuple::new(py, [origin, py.None().into_bound(py)])?.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(migration_planning_database, module)?)?; - module.add_function(wrap_pyfunction!(migration_project_schemas, module)?)?; - module.add_function(wrap_pyfunction!(migration_require_same_database, module)?)?; - module.add_function(wrap_pyfunction!(migration_resolve_origin, module)?) + Ok((origin, py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs index cf135f6f12..cec042fd6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs @@ -36,7 +36,7 @@ fn read_shape(shape: &Bound<'_, PyAny>) -> PyResult { /// Named projections of one plain SELECT, or None when unknowable. #[pyfunction] #[pyo3(signature = (*, query_sql, dialect))] -fn parse_query_shape<'py>( +pub(crate) fn parse_query_shape<'py>( py: Python<'py>, query_sql: &str, dialect: Option<&str>, @@ -125,7 +125,10 @@ pub(crate) fn explain( /// Whether output renaming explains the whole query change. #[pyfunction] #[pyo3(signature = (**options))] -fn renames_explain_change(py: Python<'_>, options: Option>) -> PyResult { +pub(crate) fn renames_explain_change( + py: Python<'_>, + options: Option>, +) -> PyResult { let arguments = KeywordArguments::read( py, "renames_explain_change", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 576f52c48e..c0f4e37068 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -3,16 +3,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - super::discovery::register(module)?; super::compatibility::register(module)?; - super::matching::register(module)?; super::old_names::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; - super::model_planning::register(module)?; - super::planning_boundary::register(module)?; - super::planning_origin::register(module)?; - super::planning_execution::register(module)?; super::projections::register(module)?; module.add_class::()?; super::fingerprint::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index ad6d16ca7e..14c1d73c87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -1,5 +1,7 @@ //! Read-only warehouse evidence gathered once for all model migrations in one plan. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{ @@ -17,8 +19,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::constants::{ GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; use crate::bindings::_helpers::state_rows::migration_objects::layout; @@ -174,8 +175,8 @@ impl MigrationStateInspection { .call_method1("get", (key,))?; let latest: Bound<'py, PyAny> = if known.is_none() { let adapter: &Bound<'py, PyAny> = self.adapter.bind(py); - let fingerprint_set: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ReadLatestFingerprints)?, + let fingerprint_set: Bound<'py, PyAny> = read_latest_fingerprints(Some(&keywords( + py, &[ ("connection", self.connection.bind(py).clone()), ("execute", adapter.getattr("execute")?), @@ -191,7 +192,7 @@ impl MigrationStateInspection { adapter.getattr("render_read_latest_fingerprints_sql")?, ), ], - )?; + )?))?; let by_identity: Bound<'py, PyAny> = fingerprint_set.getattr("fingerprints_by_identity")?; if by_identity.is_truthy()? { @@ -578,3 +579,17 @@ impl MigrationStateInspection { PyTuple::new(py, orphans) } } + +/// A fresh live-state inspection over the runtime's connection and planning database. +pub(super) fn inspection<'py>( + runtime: &Bound<'py, PyAny>, + known_fingerprints: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let state = MigrationStateInspection::new( + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, + &super::planning_origin::planning_database(runtime)?, + known_fingerprints, + )?; + Ok(Bound::new(runtime.py(), state)?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 4372fa93b3..645b9cb43c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -1,11 +1,17 @@ //! Selected-scope buildability validation before change planning. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_identity::graph::build_execution_edge_origins; +use crate::bindings::_helpers::planner_identity::graph::check_buildability; +use crate::bindings::_helpers::planner_identity::graph::missing_upstream_message; use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; -use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; -use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -19,7 +25,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( ) -> PyResult<()> { let py = runtime.py(); let project = runtime_field(runtime, |runtime| &runtime.project)?; - let selected_scope = scopes.getattr("selected_scope")?; + let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let mut external_seeds: Vec<_> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { let seed = seed?; @@ -27,38 +33,35 @@ pub(crate) fn check_selected_scope_buildability<'py>( external_seeds.push(seed.getattr("key")?); } } - let missing = native( + let missing = object( py, - NativeAttr::CheckBuildability, - &[ - ("selected_keys", selected_scope.getattr("selected_keys")?), - ("upstream_deps", selected_scope.getattr("upstream_deps")?), - ( - "snapshot", - warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, - ), - ( - "deferred_relations", - deferral.getattr("deferred_relations")?, - ), - ("satisfied_keys", frozenset(py, &external_seeds)?), - ], + check_buildability( + py, + Some(keywords( + py, + &[ + ("selected_keys", selected_scope.getattr("selected_keys")?), + ("upstream_deps", selected_scope.getattr("upstream_deps")?), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), + ( + "deferred_relations", + deferral.getattr("deferred_relations")?, + ), + ("satisfied_keys", frozenset(py, &external_seeds)?), + ], + )?), + )?, )?; if missing.is_truthy()? { - let message: String = native( + let message: String = object( py, - NativeAttr::MissingUpstreamMessage, - &[ - ("missing", missing), - ( - "edge_origins", - native( - py, - NativeAttr::BuildExecutionEdgeOrigins, - &[("project", project.clone())], - )?, - ), - ], + missing_upstream_message( + &missing, + &object(py, build_execution_edge_origins(&project)?)?, + )?, )? .extract()?; return Err(planner_error( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs new file mode 100644 index 0000000000..7525a1875b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/contexts.rs @@ -0,0 +1,49 @@ +//! Planner scope and identity contexts: built from keywords and read by planner phases. + +use pyo3::prelude::{Bound, Py, PyAny, PyResult}; +use pyo3::pymethods; +use pyo3::{PyClass, pyclass::boolean_struct::True}; + +use crate::bindings::_helpers::planner_phases::models::{ + PlannerIdentityContext, PlannerScopeResolution, +}; + +#[pymethods] +impl PlannerScopeResolution { + #[new] + #[pyo3(signature = (*, selected_scope, stale_warning_scope, inspection_scope))] + fn new( + selected_scope: Py, + stale_warning_scope: Py, + inspection_scope: Py, + ) -> Self { + Self { + selected_scope, + stale_warning_scope, + inspection_scope, + } + } +} + +#[pymethods] +impl PlannerIdentityContext { + #[new] + #[pyo3(signature = (*, version_identities, stale_warning_identities))] + fn new(version_identities: Py, stale_warning_identities: Py) -> Self { + Self { + version_identities, + stale_warning_identities, + } + } +} + +/// One field of a frozen native planner class passed as a Python object. +pub(crate) fn class_field<'py, T>( + value: &Bound<'py, PyAny>, + field: fn(&T) -> &Py, +) -> PyResult> +where + T: PyClass + Sync, +{ + Ok(field(value.cast::()?.get()).bind(value.py()).clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 460fcb822d..76370f1dca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,14 +1,19 @@ //! Write-hash reconciliation and plan entry construction. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; -use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, RelationsRequest}; +use crate::bindings::_helpers::planner_entries::models::{ + PlanEntriesRequest, PlanEntryBuildInputs, RelationsRequest, +}; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_identity::identities::merge_recomputed_model_changes; +use crate::bindings::_helpers::planner_identity::identities::with_honest_model_write_hashes; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; -use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; @@ -20,27 +25,24 @@ pub(crate) fn reconcile_execution_changes<'py>( changes: &Bound<'py, PyAny>, ) -> PyResult> { let py = execution_scope.py(); - let honest = native( + let honest = with_honest_model_write_hashes( py, - NativeAttr::WithHonestModelWriteHashes, - &[ - ("scope", execution_scope.clone()), - ("snapshot", snapshot.clone()), - ("changes", changes.clone()), - ( - "version_identities", - identities.getattr("version_identities")?, - ), - ], - )?; - let resolved = native( - py, - NativeAttr::MergeRecomputedModelChanges, - &[ - ("resolved_actions", resolved_actions.clone()), - ("changes", honest.clone()), - ], + Some(keywords( + py, + &[ + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), + ("changes", changes.clone()), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?, + ), + ], + )?), )?; + let resolved = merge_recomputed_model_changes(resolved_actions, &honest)?; Ok(ChangeReconciliation { changes: honest, resolved_actions: resolved, @@ -99,52 +101,37 @@ pub(crate) fn build_planner_entry_results<'py>( max_override.clone() }; let limit_action = if config.is_none() { - native(py, RecordId::PlanEntryBuildInputs, &[])?.getattr("microbatch_limit_action")? + default_microbatch_limit_action(py)? } else { config .getattr("microbatches")? .getattr("limits")? .getattr("action")? }; - let build_inputs = native( + let build_inputs = Bound::new( py, - RecordId::PlanEntryBuildInputs, - &[ - ("source_freshness_blocked_model_names", blocked), - ( - "external_blocked_model_names", - PyFrozenSet::new( - py, - overrides - .getattr("external_blocked_model_names")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ( - "future_cursor_config", - config_or(&config, &["cursors", "future"], none())?, - ), - ( - "start_cursor_config", - config_or(&config, &["cursors", "start"], none())?, - ), - ( - "invocation_time", - runtime_field(runtime, |runtime| &runtime.invocation_time)?, - ), - ("max_microbatches", max_microbatches), - ( - "max_microbatches_is_override", - (!max_override.is_none()) - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ("microbatch_limit_action", limit_action), - ], - )?; + PlanEntryBuildInputs { + source_freshness_blocked_model_names: blocked.unbind(), + external_blocked_model_names: PyFrozenSet::new( + py, + overrides + .getattr("external_blocked_model_names")? + .try_iter()? + .collect::>>()?, + )? + .into_any() + .unbind(), + start_cursor_override: none().unbind(), + end_cursor_override: none().unbind(), + future_cursor_config: config_or(&config, &["cursors", "future"], none())?.unbind(), + start_cursor_config: config_or(&config, &["cursors", "start"], none())?.unbind(), + invocation_time: runtime_field(runtime, |runtime| &runtime.invocation_time)?.unbind(), + max_microbatches: max_microbatches.unbind(), + max_microbatches_is_override: !max_override.is_none(), + microbatch_limit_action: limit_action.unbind(), + }, + )? + .into_any(); plan_entries(PlanEntriesRequest { project, adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 8d08fb586b..a1166b854f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,6 +1,9 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; @@ -34,20 +37,26 @@ pub(crate) fn build_planner_identity_context<'py>( scopes: &Bound<'py, PyAny>, include_stale_warning_identities: bool, ) -> PyResult> { - let identities = version_identities(project, scopes.getattr("inspection_scope")?)?; + let identities = version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + )?; let stale = if include_stale_warning_identities { - version_identities(project, scopes.getattr("stale_warning_scope")?)? + version_identities( + project, + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + )? } else { identities.clone() }; - native( + Ok(Bound::new( project.py(), - RecordId::PlannerIdentityContext, - &[ - ("version_identities", identities), - ("stale_warning_identities", stale), - ], - ) + PlannerIdentityContext { + version_identities: identities.unbind(), + stale_warning_identities: stale.unbind(), + }, + )? + .into_any()) } fn reusable_model_names( @@ -59,8 +68,12 @@ fn reusable_model_names( if !query_change_tracking { return Ok(names); } - let execution = identities.getattr("version_identities")?; - let stale = identities.getattr("stale_warning_identities")?; + let execution = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; + let stale = class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?; for name in execution_changes.getattr("models")?.try_iter()? { let name = name?; let same = |field: &str| -> PyResult { @@ -122,7 +135,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( .getattr("query_change_tracking")? .is_truthy()?, )?; - let stale_scope = scopes.getattr("stale_warning_scope")?; + let stale_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?; let mut remaining: Vec<_> = Vec::new(); for key in stale_scope .getattr("all_keys")? @@ -136,7 +149,9 @@ pub(crate) fn detect_stale_warning_changes<'py>( remaining.push(key); } } - let stale_identities = identities.getattr("stale_warning_identities")?; + let stale_identities = class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?; let remaining_changes = native( py, NativeAttr::DetectModelChangesInScope, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index b75082aa1c..79bb897f9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod buildability; pub(crate) mod constants; +pub(crate) mod contexts; pub(crate) mod execution_changes; pub(crate) mod full_refresh; pub(crate) mod identities; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index 3df8d67d83..baecdfe807 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -51,3 +51,20 @@ pub(crate) struct PlannerWarehouseState { pub(crate) old_name_views: Py, pub(crate) migration_fingerprints: Py, } + +/// Resolved planner scopes for one plan build. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerScopeResolution { + pub(crate) selected_scope: Py, + pub(crate) stale_warning_scope: Py, + pub(crate) inspection_scope: Py, +} + +/// Expected version identities for inspection and stale-warning scopes. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerIdentityContext { + pub(crate) version_identities: Py, + pub(crate) stale_warning_identities: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index 81a56d9ab4..1ee9b4fd85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::records::ids::{ErrorId, NativeItem, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyFrozenSet, PyString, PyTuple}; +use pyo3::{BoundObject, IntoPyObject}; /// A Python `str`. pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { @@ -181,3 +182,12 @@ pub(crate) fn model_key<'py>( ], ) } + +/// A native helper's Rust result as a Python object. +pub(crate) fn object<'py, T>(py: Python<'py>, value: T) -> PyResult> +where + T: IntoPyObject<'py>, + PyErr: From, +{ + Ok(value.into_pyobject(py)?.into_bound().into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index e310d807c3..381a50a5a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,14 +2,22 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; +use crate::bindings::_helpers::planner_identity::staleness::build_stale_out_of_selection_warnings; +use crate::bindings::_helpers::planner_migrations::fingerprint::with_migration_fingerprints; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ ChangeReconciliation, ModelEntryResults, PlanOutputFields, }; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ native, replace, sorted_names, strings, text, tuple, }; +use crate::bindings::_helpers::planner_phases::retention::plan_retention; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; @@ -75,7 +83,9 @@ pub(crate) fn base_plan_output<'py>( planned_sql_tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let version = identities.getattr("version_identities")?; + let version = class_field(identities, |i: &PlannerIdentityContext| { + &i.version_identities + })?; let extras = native( py, RecordId::PlanOutputExtras, @@ -170,23 +180,30 @@ pub(crate) fn add_storage_policies<'py>( let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } - entries = native( + entries = object( py, - NativeAttr::WithMigrationFingerprints, - &[ - ("entries", entries), - ("models_by_name", models.into_any()), - ( - "fingerprints", - warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, - ), - ( - "dialect", - runtime - .getattr("adapter")? - .call_method0("sql_analysis_dialect")?, - ), - ], + with_migration_fingerprints( + py, + Some(keywords( + py, + &[ + ("entries", entries), + ("models_by_name", models.into_any()), + ( + "fingerprints", + warehouse_field(warehouse, |warehouse| { + &warehouse.migration_fingerprints + })?, + ), + ( + "dialect", + runtime + .getattr("adapter")? + .call_method0("sql_analysis_dialect")?, + ), + ], + )?), + )?, )?; } let entries = with_column_rename_hints( @@ -206,14 +223,17 @@ pub(crate) fn add_storage_policies<'py>( output.field("warnings")?, ], )?; - let selected = scopes.getattr("selected_scope")?; - let storage = [ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scope", selected), - ]; - let table_types = native(py, NativeAttr::PlanTableTypes, &storage)?; - let retention = native(py, NativeAttr::PlanRetention, &storage)?; + let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; + let table_types = native( + py, + NativeAttr::PlanTableTypes, + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scope", selected.clone()), + ], + )?; + let retention = plan_retention(runtime, warehouse, &selected)?; output.set("model_entries", entries); output.set("migration_entries", migration_entries); output.set("column_migration_entries", column_entries); @@ -245,31 +265,44 @@ pub(crate) fn add_plan_warnings<'py>( if !policies.getattr("selection_diagnostics")?.is_truthy()? { return Ok(output); } - let stale = native( + let stale = object( py, - NativeAttr::BuildStaleOutOfSelectionWarnings, - &[ - ("original_scope", scopes.getattr("stale_warning_scope")?), - ("execution_scope", scopes.getattr("inspection_scope")?), - ("changes", stale_warning_changes.clone()), - ( - "snapshot", - warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, - ), - ( - "version_identities", - identities.getattr("stale_warning_identities")?, - ), - ("source_freshness", source_freshness.clone()), - ( - "include_sources", - PyBool::new(py, false).to_owned().into_any(), - ), - ( - "model_changes_complete", - PyBool::new(py, true).to_owned().into_any(), - ), - ], + build_stale_out_of_selection_warnings( + py, + Some(keywords( + py, + &[ + ( + "original_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)?, + ), + ( + "execution_scope", + class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, + ), + ("changes", stale_warning_changes.clone()), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), + ( + "version_identities", + class_field(identities, |i: &PlannerIdentityContext| { + &i.stale_warning_identities + })?, + ), + ("source_freshness", source_freshness.clone()), + ( + "include_sources", + PyBool::new(py, false).to_owned().into_any(), + ), + ( + "model_changes_complete", + PyBool::new(py, true).to_owned().into_any(), + ), + ], + )?), + )?, )?; if stale.is_truthy()? { let warnings = concat(py, &[output.field("warnings")?, stale])?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 3ed0682940..5092e097f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -6,6 +6,8 @@ use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!( super::scopes::resolve_planner_scopes, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index e58fa52e5b..39a084effd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::constants::{ @@ -14,7 +15,7 @@ use crate::bindings::_helpers::planner_phases::objects::{ planner_model, replace, sorted_names, text, tuple, }; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -39,15 +40,7 @@ pub(crate) fn effective_target_config<'py>( if project_config.is_none() || local_config.is_none() || target_name.is_none() { return native(py, RecordId::TargetConfig, &[]); } - native( - py, - NativeAttr::ResolveTargetConfig, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name), - ], - ) + resolve_target_config(py, &project_config, &local_config, &target_name) } /// A deterministic identifier-fitted conversion copy name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs index 96c8604552..6259243ca5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs @@ -1,14 +1,13 @@ //! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::{PyDict, PyTuple}; use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; #[pymethods] impl PlannerRuntime { @@ -97,9 +96,7 @@ impl PlannerWarehouseState { migration_fingerprints: if args.passed("migration_fingerprints")? { args.value("migration_fingerprints")?.unbind() } else { - registry::native(py, NativeAttr::MigrationFingerprintCache)? - .call0()? - .unbind() + Py::new(py, MigrationFingerprintCache::open(py, None)?)?.into_any() }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 2a4666fb15..0fd177e226 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,9 +1,12 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::IntoPyObject; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{replace, tuple}; +use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -17,47 +20,48 @@ pub(crate) fn resolve_planner_scopes<'py>( ) -> PyResult> { let py = project.py(); let auto_load = policies.getattr("auto_load_sources")?; - let selected = native( + let selected = build_planner_scope( py, - NativeAttr::BuildPlannerScope, - &[ - ("project", project.clone()), - ("select", selection.getattr("select")?), - ("exclude", selection.getattr("exclude")?), - ("auto_load_sources", auto_load.clone()), - ("selected_keys", selection.getattr("selected_keys")?), - ( - "python_read_source_names", - selection.getattr("python_read_source_names")?, - ), - ( - "sql_test_selection", - selection.getattr("sql_test_selection")?, - ), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("select", selection.getattr("select")?), + ("exclude", selection.getattr("exclude")?), + ("auto_load_sources", auto_load.clone()), + ("selected_keys", selection.getattr("selected_keys")?), + ( + "python_read_source_names", + selection.getattr("python_read_source_names")?, + ), + ( + "sql_test_selection", + selection.getattr("sql_test_selection")?, + ), + ], + )?), )?; let empty = PyTuple::empty(py).into_any(); - let full = native( + let full = build_planner_scope( py, - NativeAttr::BuildPlannerScope, - &[ - ("project", project.clone()), - ("select", empty.clone()), - ("exclude", empty), - ("auto_load_sources", auto_load), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("select", empty.clone()), + ("exclude", empty), + ("auto_load_sources", auto_load), + ], + )?), )?; - let stale_keys = native( + let stale_keys = object( py, - NativeAttr::TransitiveClosureMany, - &[ - ("starts", selected.getattr("selected_keys")?), - ("edges", full.getattr("upstream_deps")?), - ( - "include_starts", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + transitive_closure_many( + &selected.getattr("selected_keys")?, + &full.getattr("upstream_deps")?, + true, + None, + )?, )?; let all_keys = PyDict::new(py); for (name, key) in full.getattr("all_keys")?.downcast::()?.iter() { @@ -91,13 +95,13 @@ pub(crate) fn resolve_planner_scopes<'py>( ), ], )?; - native( + Ok(Bound::new( py, - RecordId::PlannerScopeResolution, - &[ - ("selected_scope", selected.clone()), - ("stale_warning_scope", stale_scope), - ("inspection_scope", selected), - ], - ) + PlannerScopeResolution { + selected_scope: selected.clone().unbind(), + stale_warning_scope: stale_scope.unbind(), + inspection_scope: selected.unbind(), + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index 8627463617..47802ad3e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -8,7 +8,9 @@ use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, MigrationPhase, }; use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; @@ -46,7 +48,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( .into_bound(py) .into_any(); progress(runtime, "Inspecting warehouse state...")?; - let selected_scope = scopes.getattr("selected_scope")?; + let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( @@ -76,8 +78,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let stale_keys = pyo3::types::PyFrozenSet::new( py, - scopes - .getattr("stale_warning_scope")? + class_field(scopes, |s: &PlannerScopeResolution| &s.stale_warning_scope)? .getattr("all_keys")? .call_method0("values")? .try_iter()? @@ -109,7 +110,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( project: project.clone(), adapter: adapter.clone(), connection: runtime_field(runtime, |runtime| &runtime.connection)?, - scope: scopes.getattr("inspection_scope")?, + scope: class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, deferral: deferral.clone(), project_config: config, local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index d419863cd5..9cbc14ad68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -52,21 +52,6 @@ const R_PLANNER_SCOPE: RecordSpec = RecordSpec { ), ], }; -const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerRelationsContext", - doc: "Resolved relation and source inputs for plan entry construction.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_read_map"), - FieldSpec::required("python_source_read_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), - ], -}; const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { name: "SqlTestAssertionStep", doc: "One zero-row assertion step in a SQL-native unit test.", @@ -935,7 +920,6 @@ pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ R_PATH_SELECTOR, R_SQL_TEST_SELECTION, R_PLANNER_SCOPE, - R_PLANNER_RELATIONS_CONTEXT, R_SQL_TEST_ASSERTION_STEP, R_PLANNER_SELECTION, R_PLANNER_OVERRIDES, @@ -1158,27 +1142,6 @@ const R_UNIT_TEST_SELECTOR_SPLIT: RecordSpec = RecordSpec { FieldSpec::required("sql_test_selection"), ], }; -const R_MODEL_PLAN_CONTEXT: RecordSpec = RecordSpec { - name: "ModelPlanContext", - doc: "Relation lookups and source columns for building one model plan entry.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("models_by_name"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("functions_by_name", FieldDefault::EmptyDict), - FieldSpec::new( - "runtime_cursor_producer_names", - FieldDefault::EmptyFrozenset, - ), - FieldSpec::new("future_cursor_config", FieldDefault::None), - FieldSpec::new("start_cursor_config", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::None), - ], -}; const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { name: "CursorOverridePair", doc: "Resolved per-model cursor start/end overrides for plan entry construction.", @@ -1187,28 +1150,6 @@ const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { FieldSpec::new("end_cursor_override", FieldDefault::None), ], }; -const R_PLAN_ENTRY_BUILD_INPUTS: RecordSpec = RecordSpec { - name: "PlanEntryBuildInputs", - doc: "Blocked models and cursor overrides for plan entry building.", - fields: &[ - FieldSpec::new( - "source_freshness_blocked_model_names", - FieldDefault::EmptyFrozenset, - ), - FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyFrozenset), - FieldSpec::new("start_cursor_override", FieldDefault::None), - FieldSpec::new("end_cursor_override", FieldDefault::None), - FieldSpec::new("future_cursor_config", FieldDefault::None), - FieldSpec::new("start_cursor_config", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::None), - FieldSpec::new("max_microbatches", FieldDefault::None), - FieldSpec::new("max_microbatches_is_override", FieldDefault::False), - FieldSpec::new( - "microbatch_limit_action", - FieldDefault::NativeMember(RecordId::MicrobatchLimitAction, "ERROR"), - ), - ], -}; const R_FUNCTION_CHANGE_RESULT: RecordSpec = RecordSpec { name: "FunctionChangeResult", doc: "Per-function output from change detection.", @@ -1394,62 +1335,6 @@ const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { FieldSpec::new("annotation_module", FieldDefault::None), ], }; -const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { - name: "PlannerScopeResolution", - doc: "Resolved planner scopes for one plan build.", - fields: &[ - FieldSpec::required("selected_scope"), - FieldSpec::required("stale_warning_scope"), - FieldSpec::required("inspection_scope"), - ], -}; -const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { - name: "ModelMigrationRequest", - doc: "One destination model whose data should come from an earlier relation.", - fields: &[ - FieldSpec::required("model"), - FieldSpec::required("discovery"), - FieldSpec::new("raw_origin", FieldDefault::None), - FieldSpec::new("origin_location", FieldDefault::None), - FieldSpec::new("origin_model", FieldDefault::None), - FieldSpec::new("force", FieldDefault::False), - FieldSpec::new("identity_only", FieldDefault::False), - ], -}; -const R_MODEL_MIGRATION_DECLARATION: RecordSpec = RecordSpec { - name: "ModelMigrationDeclaration", - doc: "One project model's explicit migrate_from, resolved to a relation.", - fields: &[ - FieldSpec::required("model_name"), - FieldSpec::required("origin_location"), - FieldSpec::required("origin_model"), - ], -}; -const R_MODEL_MIGRATION_DISCOVERY: RecordSpec = RecordSpec { - name: "ModelMigrationDiscovery", - doc: "Manual and automatically discovered migration requests for one plan.", - fields: &[ - FieldSpec::new("requests", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - FieldSpec::new("destination_fingerprints", FieldDefault::EmptyDict), - ], -}; -const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { - name: "MigrationCompatibilityResult", - doc: "Compatibility status plus human-readable blocking findings.", - fields: &[ - FieldSpec::required("status"), - FieldSpec::new("findings", FieldDefault::EmptyTuple), - ], -}; -const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerIdentityContext", - doc: "Expected version identities for inspection and stale-warning scopes.", - fields: &[ - FieldSpec::required("version_identities"), - FieldSpec::required("stale_warning_identities"), - ], -}; /// Planner value records, in registration order. pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1469,9 +1354,7 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_BACKFILL_RESULT, R_CHANGE_DETECTION_RESULT, R_UNIT_TEST_SELECTOR_SPLIT, - R_MODEL_PLAN_CONTEXT, R_CURSOR_OVERRIDE_PAIR, - R_PLAN_ENTRY_BUILD_INPUTS, R_FUNCTION_CHANGE_RESULT, R_PLANNER_CHANGE_RESULTS, R_RESOLVED_MODEL_ACTION, @@ -1490,12 +1373,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_SCENARIO_ARTIFACT_IDENTITY, R_SCENARIO_ARTIFACT_NAME, R_PLAN_PROVIDER_USAGE, - R_PLANNER_SCOPE_RESOLUTION, - R_MODEL_MIGRATION_REQUEST, - R_MODEL_MIGRATION_DECLARATION, - R_MODEL_MIGRATION_DISCOVERY, - R_MIGRATION_COMPATIBILITY_RESULT, - R_PLANNER_IDENTITY_CONTEXT, ]; /// Planner string enums, in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index fa27998cc4..d0ef32f341 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -1,12 +1,14 @@ //! Snapshot conversion and cursor window policy boundary. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; use crate::bindings::_helpers::cursor::scalars::try_parse; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_resolution::maximum_safety::apply_maximum_start_policy; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -114,7 +116,7 @@ fn read_policy<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn apply_typed_cursor_replay_policy<'py>( +pub(crate) fn apply_typed_cursor_replay_policy<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -168,7 +170,7 @@ pub(super) fn required_scalar<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_cursor_bounds<'py>( +pub(crate) fn compute_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -257,10 +259,7 @@ fn compute_cursor_bounds<'py>( .into_any(), ), ]); - let replayed = call_with( - ®istry::native(py, NativeAttr::ApplyTypedCursorReplayPolicy)?, - &replay_args, - )?; + let replayed = apply_typed_cursor_replay_policy(py, Some(keywords(py, &replay_args)?))?; let bounds = call_with( &bounds_class, &[("start", replayed), ("end", session.python_scalar(&end)?)], @@ -292,10 +291,7 @@ fn compute_cursor_bounds<'py>( ] { policy_args.push((key, args.value(key)?)); } - call_with( - ®istry::native(py, NativeAttr::ApplyMaximumStartPolicy)?, - &policy_args, - ) + apply_maximum_start_policy(py, Some(keywords(py, &policy_args)?)) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 6d8260bb67..897166f1a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,20 +1,33 @@ //! Model cursor orchestration over live warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; +use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; +use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_cursor_window; +use crate::bindings::_helpers::planner_resolution::cursor_windows::empty_input_rebuild_refusal; +use crate::bindings::_helpers::planner_resolution::cursor_windows::unreadable_cursor_inputs_error; +use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; +use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::{IntoPyObject, PyErr}; @@ -76,22 +89,28 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(py.None().into_bound(py)); } let incremental_mode = config(&model, "incremental_mode")?; - let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; - let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; + let future_config = future_cursor_config( + &model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?; + let start_config = start_cursor_config( + &model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?; if incremental_mode.eq("microbatch")? { return unbounded(py); } let cursor_type = config(&model, "cursor_type")?; let cursor_grain = config(&model, "cursor_grain")?; let (start_override, end_override) = request.cursor_overrides.clone(); - let bounded = call_with( - ®istry::native(py, NativeAttr::ResolveBoundedCursorOverride)?, - &[ - ("start_cursor_override", start_override.clone()), - ("end_cursor_override", end_override.clone()), - ("cursor_type", cursor_type.clone()), - ("cursor_grain", cursor_grain.clone()), - ], + let bounded = object( + py, + resolve_bounded_cursor_override( + &start_override, + &end_override, + &cursor_type, + &cursor_grain, + )?, )?; if !bounded.is_none() { return Ok(bounded); @@ -100,11 +119,10 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? .call_method1("get", (model_reads::name_object(&model)?,))?; - let roles = call_with( - ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, - &[("model", model.clone())], - )?; - let producers = context.getattr("runtime_cursor_producer_names")?; + let roles = resolve_cursor_input_roles(&model)?; + let producers = class_field(context, |c: &ModelPlanContext| { + &c.runtime_cursor_producer_names + })?; let mut runtime_owned = false; for name in roles.getattr("watermark_inputs")?.try_iter()? { if producers.contains(name?)? { @@ -115,23 +133,19 @@ pub(crate) fn compute_model_cursor_bounds<'py>( let full_refresh = request.full_refresh; let replaces = request.replaces_relation; if !full_refresh && !runtime_owned && !cursor_snapshot.is_none() { - let empty = call_with( - ®istry::native(py, NativeAttr::CursorInputsWithoutRows)?, - &[("cursor_snapshot", cursor_snapshot.clone())], - )? - .getattr("input_names")?; + let empty = cursor_inputs_without_rows(&cursor_snapshot)?.getattr("input_names")?; if empty.is_truthy()? && replaces && (destination_exists(&model, snapshot)? || history_redirected(&model, &cursor_snapshot)?) { - let refusal = call_with( - ®istry::native(py, NativeAttr::EmptyInputRebuildRefusal)?, - &[ - ("model_name", model_reads::name_object(&model)?), - ("input_names", empty.clone()), - ("cursor_type", cursor_type.clone()), - ], + let refusal = object( + py, + empty_input_rebuild_refusal( + &model_reads::name_object(&model)?.extract::()?, + empty.extract::>()?, + cursor_type.extract::>()?.as_deref(), + ), )?; return Err(PyErr::from_value(call_with( ®istry::error(py, ErrorId::PlannerInput)?, @@ -143,13 +157,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( )?)); } if empty.is_truthy()? { - return call_with( - ®istry::native(py, NativeAttr::EmptyCursorWindow)?, - &[ - ("cursor_type", cursor_type), - ("cursor_start", config(&model, "cursor_start")?), - ], - ); + return empty_cursor_window(&cursor_type, &config(&model, "cursor_start")?); } if !cursor_snapshot .getattr("watermarks_available")? @@ -161,13 +169,11 @@ pub(crate) fn compute_model_cursor_bounds<'py>( } else { cursor_snapshot.getattr("unavailable_watermark_tags")? }; - return Err(PyErr::from_value(call_with( - ®istry::native(py, NativeAttr::UnreadableCursorInputsError)?, - &[ - ("model_name", model_reads::name_object(&model)?), - ("cursor_type", cursor_type), - ("input_names", names), - ], + return Err(PyErr::from_value(unreadable_cursor_inputs_error( + py, + &model_reads::name_object(&model)?.extract::()?, + cursor_type.extract::>()?.as_deref(), + names.extract::>()?, )?)); } } @@ -181,10 +187,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( return Ok(py.None().into_bound(py)); } if replaces { - cursor_snapshot = call_with( - ®istry::native(py, NativeAttr::WithoutDestinationCursor)?, - &[("cursor_snapshot", cursor_snapshot)], - )?; + cursor_snapshot = without_destination_cursor(&cursor_snapshot)?; } let backfill = &request.backfill; let duration = if backfill.getattr("action")?.eq("bounded")? { @@ -196,7 +199,10 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ®istry::class(py, RecordId::MaximumStartPolicyInputs)?, &[ ("config", start_config), - ("invocation_time", context.getattr("invocation_time")?), + ( + "invocation_time", + class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, + ), ( "incremental_strategy", config(&model, "incremental_strategy")?, @@ -204,43 +210,52 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ("incremental_mode", incremental_mode), ], )?; - let bounds = call_with( - ®istry::native(py, NativeAttr::ComputeCursorBounds)?, - &[ - ("cursor_snapshot", cursor_snapshot.clone()), - ("cursor_type", cursor_type.clone()), - ("cursor_start", config(&model, "cursor_start")?), - ("lookback", config(&model, "lookback")?), - ("backfill_duration", duration), - ("start_cursor_override", start_override.clone()), - ("end_cursor_override", end_override.clone()), - ( - "is_microbatch", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ("cursor_grain", cursor_grain.clone()), - ("maximum_start_policy", policy), - ], + let bounds = compute_cursor_bounds( + py, + Some(keywords( + py, + &[ + ("cursor_snapshot", cursor_snapshot.clone()), + ("cursor_type", cursor_type.clone()), + ("cursor_start", config(&model, "cursor_start")?), + ("lookback", config(&model, "lookback")?), + ("backfill_duration", duration), + ("start_cursor_override", start_override.clone()), + ("end_cursor_override", end_override.clone()), + ( + "is_microbatch", + false.into_pyobject(py)?.to_owned().into_any(), + ), + ("cursor_grain", cursor_grain.clone()), + ("maximum_start_policy", policy), + ], + )?), )?; if bounds.is_none() { return Ok(bounds); } - call_with( - ®istry::native(py, NativeAttr::ResolveFutureCursorSafety)?, - &[ - ("bounds", bounds), - ("cursor_type", cursor_type), - ("cursor_grain", cursor_grain), - ("config", future_config), - ("invocation_time", context.getattr("invocation_time")?), - ( - "has_complete_override", - (!start_override.is_none() && !end_override.is_none()) - .into_pyobject(py)? - .to_owned() - .into_any(), - ), - ("input_evidence", cursor_snapshot.getattr("input_evidence")?), - ], + resolve_future_cursor_safety( + py, + Some(keywords( + py, + &[ + ("bounds", bounds), + ("cursor_type", cursor_type), + ("cursor_grain", cursor_grain), + ("config", future_config), + ( + "invocation_time", + class_field(context, |c: &ModelPlanContext| &c.invocation_time)?, + ), + ( + "has_complete_override", + (!start_override.is_none() && !end_override.is_none()) + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ("input_evidence", cursor_snapshot.getattr("input_evidence")?), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs index 8d61ea65dc..c066e4a339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -93,7 +93,7 @@ fn resolve_future_cursor_config<'py>( /// Resolve automatic-start policy with model config taking precedence. #[pyfunction] #[pyo3(signature = (*, model, project_config))] -fn resolve_start_cursor_config<'py>( +pub(crate) fn resolve_start_cursor_config<'py>( model: &Bound<'py, PyAny>, project_config: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index da8a171299..e765b7856f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -1,22 +1,35 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::cursor::conversion::CursorSession; use crate::bindings::_helpers::cursor::scalars::{parse_value, sentinel_to_token}; use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; +use crate::bindings::_helpers::model_config::config_values::config_cursor_bound; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::plan_output::display::resolve_bounded_cursor_override; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::compute_cursor_bounds; use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; +use crate::bindings::_helpers::planner_resolution::cursor_windows::cursor_inputs_without_rows; +use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::without_destination_cursor; +use crate::bindings::_helpers::planner_resolution::future_safety::resolve_future_cursor_safety; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::resolve_microbatch_lookback; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; @@ -65,14 +78,6 @@ pub(crate) struct CursorWindowInputs<'py> { pub(crate) cursor_input_relations: Bound<'py, PyAny>, } -fn native<'py>( - py: Python<'py>, - name: NativeAttr, - pairs: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - call_with(®istry::native(py, name)?, pairs) -} - /// One string-valued model configuration entry. pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { Ok( @@ -86,13 +91,9 @@ pub(crate) fn config_bound<'py>( model: &Bound<'py, PyAny>, key: &str, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::GetConfigCursorBound, - &[ - ("values", model.getattr("config")?.getattr("values")?), - ("key", string(model.py(), key)), - ], + config_cursor_bound(&model.getattr("config")?.getattr("values")?, key)?, ) } @@ -182,10 +183,9 @@ fn rolling_window_bounds<'py>( ) -> PyResult> { let py = model.py(); let grain = config_str(model, "cursor_grain")?; - let lookback = native( + let lookback = object( py, - NativeAttr::ResolveMicrobatchLookback, - &[("model", model.clone()), ("effective_grain", grain.clone())], + resolve_microbatch_lookback(model, grain.extract::>()?)?, )?; let lookback = if lookback.is_truthy()? { lookback @@ -253,24 +253,26 @@ fn apply_future_safety<'py>( if bounds.is_none() || bounds.getattr("end")?.eq(sentinel(py, "END")?)? { return Ok(bounds); } - native( + resolve_future_cursor_safety( py, - NativeAttr::ResolveFutureCursorSafety, - &[ - ("bounds", bounds), - ("cursor_type", typing.0.clone()), - ("cursor_grain", typing.1.clone()), - ("config", inputs.policies.0.clone()), - ("invocation_time", inputs.invocation_time.clone()), - ( - "has_complete_override", - boolean( - py, - !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), - )?, - ), - ("input_evidence", input_evidence), - ], + Some(keywords( + py, + &[ + ("bounds", bounds), + ("cursor_type", typing.0.clone()), + ("cursor_grain", typing.1.clone()), + ("config", inputs.policies.0.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "has_complete_override", + boolean( + py, + !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), + )?, + ), + ("input_evidence", input_evidence), + ], + )?), ) } @@ -278,15 +280,14 @@ fn bounded_override<'py>( model: &Bound<'py, PyAny>, inputs: &CursorWindowInputs<'py>, ) -> PyResult> { - native( + object( model.py(), - NativeAttr::ResolveBoundedCursorOverride, - &[ - ("start_cursor_override", inputs.overrides.0.clone()), - ("end_cursor_override", inputs.overrides.1.clone()), - ("cursor_type", config_str(model, "cursor_type")?), - ("cursor_grain", config_str(model, "cursor_grain")?), - ], + resolve_bounded_cursor_override( + &inputs.overrides.0.clone(), + &inputs.overrides.1.clone(), + &config_str(model, "cursor_type")?, + &config_str(model, "cursor_grain")?, + )?, ) } @@ -323,21 +324,23 @@ fn snapshot_bounds<'py>( ("incremental_mode", config_str(model, "incremental_mode")?), ], )?; - let bounds = native( + let bounds = compute_cursor_bounds( py, - NativeAttr::ComputeCursorBounds, - &[ - ("cursor_snapshot", request.cursor_snapshot.clone()), - ("cursor_type", request.typing.0.clone()), - ("cursor_start", config_bound(model, "cursor_start")?), - ("lookback", request.lookback), - ("backfill_duration", backfill_duration(&inputs.backfill)?), - ("start_cursor_override", inputs.overrides.0.clone()), - ("end_cursor_override", inputs.overrides.1.clone()), - ("is_microbatch", boolean(py, request.is_microbatch)?), - ("cursor_grain", request.typing.1.clone()), - ("maximum_start_policy", policy), - ], + Some(keywords( + py, + &[ + ("cursor_snapshot", request.cursor_snapshot.clone()), + ("cursor_type", request.typing.0.clone()), + ("cursor_start", config_bound(model, "cursor_start")?), + ("lookback", request.lookback), + ("backfill_duration", backfill_duration(&inputs.backfill)?), + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("is_microbatch", boolean(py, request.is_microbatch)?), + ("cursor_grain", request.typing.1.clone()), + ("maximum_start_policy", policy), + ], + )?), )?; let bounds = apply_future_safety( bounds, @@ -358,11 +361,7 @@ fn model_cursor_snapshot<'py>( } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { - native( - cursor_snapshot.py(), - NativeAttr::WithoutDestinationCursor, - &[("cursor_snapshot", cursor_snapshot)], - ) + without_destination_cursor(&cursor_snapshot) } /// Compute cursor bounds for inclusion on the plan entry. @@ -447,34 +446,19 @@ pub(crate) fn microbatch_range<'py>( for relation in inputs.cursor_input_relations.try_iter()? { grains.push(relation?.getattr("cursor_grain")?); } - let effective_grain = native( - py, - NativeAttr::ResolveEffectiveTimestampGrain, - &[ - ("cursor_type", cursor_type.clone()), - ("downstream_grain", config_str(model, "cursor_grain")?), - ("microbatch_strategy", strategy), - ("cursor_input_grains", PyTuple::new(py, grains)?.into_any()), - ], + let effective_grain = resolve_effective_timestamp_grain( + &cursor_type, + &config_str(model, "cursor_grain")?, + &PyTuple::new(py, grains)?.into_any(), + strategy.extract::>()?.as_deref(), )?; if cursor_type.eq(TIMESTAMP)? && !effective_grain.is_none() { - cursor_snapshot = native( - py, - NativeAttr::NormalizeCursorSnapshotGrain, - &[ - ("cursor_snapshot", cursor_snapshot), - ("cursor_type", cursor_type.clone()), - ("effective_grain", effective_grain.clone()), - ], - )?; + cursor_snapshot = + normalize_cursor_snapshot_grain(&cursor_snapshot, &cursor_type, &effective_grain)?; } - let lookback = native( + let lookback = object( py, - NativeAttr::ResolveMicrobatchLookback, - &[ - ("model", model.clone()), - ("effective_grain", effective_grain.clone()), - ], + resolve_microbatch_lookback(model, effective_grain.extract::>()?)?, )?; snapshot_bounds( model, @@ -509,11 +493,7 @@ pub(crate) fn empty_cursor_inputs<'py>( if cursor_snapshot.is_none() { return nothing(); } - let decision = native( - py, - NativeAttr::CursorInputsWithoutRows, - &[("cursor_snapshot", cursor_snapshot)], - )?; + let decision = cursor_inputs_without_rows(&cursor_snapshot)?; Ok(( decision.getattr("input_names")?, decision.getattr("waiting_on_empty_inputs")?.is_truthy()?, @@ -526,8 +506,14 @@ pub(crate) fn cursor_safety_configs<'py>( context: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { Ok(( - future_cursor_config(model, &context.getattr("future_cursor_config")?)?, - start_cursor_config(model, &context.getattr("start_cursor_config")?)?, + future_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.future_cursor_config)?, + )?, + start_cursor_config( + model, + &class_field(context, |c: &ModelPlanContext| &c.start_cursor_config)?, + )?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index 096a31951d..493912ad2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -41,7 +41,7 @@ fn decide_empty_cursor_inputs<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot))] -fn cursor_inputs_without_rows<'py>( +pub(crate) fn cursor_inputs_without_rows<'py>( cursor_snapshot: &Bound<'py, PyAny>, ) -> PyResult> { if cursor_snapshot @@ -74,7 +74,7 @@ fn cursor_inputs_without_rows<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type, cursor_start))] -fn empty_cursor_window<'py>( +pub(crate) fn empty_cursor_window<'py>( cursor_type: &Bound<'py, PyAny>, cursor_start: &Bound<'py, PyAny>, ) -> PyResult> { @@ -117,7 +117,7 @@ fn cursor_window_override_flags(cursor_type: Option<&str>) -> String { #[pyfunction] #[pyo3(signature = (*, model_name, input_names, cursor_type))] -fn empty_input_rebuild_refusal( +pub(crate) fn empty_input_rebuild_refusal( model_name: &str, input_names: Vec, cursor_type: Option<&str>, @@ -136,7 +136,7 @@ fn empty_input_rebuild_refusal( #[pyfunction] #[pyo3(signature = (*, model_name, cursor_type, input_names))] -fn unreadable_cursor_inputs_error<'py>( +pub(crate) fn unreadable_cursor_inputs_error<'py>( py: Python<'py>, model_name: &str, cursor_type: Option<&str>, @@ -172,7 +172,7 @@ fn unreadable_cursor_inputs_error<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot))] -fn without_destination_cursor<'py>( +pub(crate) fn without_destination_cursor<'py>( cursor_snapshot: &Bound<'py, PyAny>, ) -> PyResult> { let py = cursor_snapshot.py(); @@ -185,7 +185,7 @@ fn without_destination_cursor<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_type, downstream_grain, cursor_input_grains, microbatch_strategy=None))] -fn resolve_effective_timestamp_grain<'py>( +pub(crate) fn resolve_effective_timestamp_grain<'py>( cursor_type: &Bound<'py, PyAny>, downstream_grain: &Bound<'py, PyAny>, cursor_input_grains: &Bound<'py, PyAny>, @@ -220,7 +220,7 @@ fn resolve_effective_timestamp_grain<'py>( #[pyfunction] #[pyo3(signature = (*, cursor_snapshot, cursor_type, effective_grain))] -fn normalize_cursor_snapshot_grain<'py>( +pub(crate) fn normalize_cursor_snapshot_grain<'py>( cursor_snapshot: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, effective_grain: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs index 81ac8cc755..67d3aa58c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -18,7 +18,7 @@ use std::cmp::Ordering; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_future_cursor_safety<'py>( +pub(crate) fn resolve_future_cursor_safety<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs index e2b3f6c066..40c12926b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/locations.rs @@ -14,17 +14,21 @@ fn locations<'py>(items: &Bound<'py, PyAny>) -> PyResult> { } #[pyfunction] -fn build_model_locations<'py>(models: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_model_locations<'py>( + models: &Bound<'py, PyAny>, +) -> PyResult> { locations(models) } #[pyfunction] -fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_seed_locations<'py>(seeds: &Bound<'py, PyAny>) -> PyResult> { locations(seeds) } #[pyfunction] -fn build_function_locations<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn build_function_locations<'py>( + functions: &Bound<'py, PyAny>, +) -> PyResult> { locations(functions) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 39b8a0efe3..7b2321512b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -1,7 +1,11 @@ //! Maximum automatic-start recovery and evidence boundary. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::apply_typed_cursor_replay_policy; use crate::bindings::_helpers::planner_resolution::cursor_bounds::{ optional_scalar, required_scalar, scalars, }; @@ -9,7 +13,7 @@ use crate::bindings::_helpers::planner_resolution::cursor_safety::{ horizon, replace, safety_duration, safety_error, timestamp_bound, utc_clock, }; use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -23,7 +27,7 @@ use std::cmp::Ordering; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn maximum_allowed_start<'py>( +pub(crate) fn maximum_allowed_start<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult { @@ -64,7 +68,7 @@ fn maximum_allowed_start<'py>( #[pyfunction] #[pyo3(signature = (**kwargs))] -fn apply_maximum_start_policy<'py>( +pub(crate) fn apply_maximum_start_policy<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -171,10 +175,7 @@ fn apply_maximum_start_policy<'py>( ] { replay.push((key, args.value(key)?)); } - call_with( - ®istry::native(py, NativeAttr::ApplyTypedCursorReplayPolicy)?, - &replay, - )? + apply_typed_cursor_replay_policy(py, Some(keywords(py, &replay)?))? } else { bounds.getattr("start")? }; @@ -247,11 +248,7 @@ fn apply_maximum_start_policy<'py>( } fn rendered(value: &Bound<'_, PyAny>) -> PyResult { - call_with( - ®istry::native(value.py(), NativeAttr::CursorRender)?, - &[("value", value.clone())], - )? - .extract() + object(value.py(), render_value(value)?)?.extract() } /// The message of an automatic start that exceeds its horizon. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index 063cba97d8..93bab4d84b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -1,20 +1,20 @@ //! Planner reference resolution with Python adapter rendering callbacks. mod constants; -mod cursor_bounds; +pub(crate) mod cursor_bounds; pub(crate) mod cursor_model; pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; mod cursor_safety; pub(crate) mod cursor_warnings; -mod cursor_windows; -mod future_safety; -mod locations; -mod maximum_safety; +pub(crate) mod cursor_windows; +pub(crate) mod future_safety; +pub(crate) mod locations; +pub(crate) mod maximum_safety; pub(crate) mod models; pub(crate) mod orchestration; -mod refs; +pub(crate) mod refs; pub(crate) mod registration; -mod sources; +pub(crate) mod sources; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 81d9bbcea8..f1860f511b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,6 +9,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; use crate::bindings::_helpers::planner_resolution::refs::{ @@ -43,11 +45,13 @@ impl<'py> Resolution<'py> { Ok(Self { adapter, external, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - function_locations: context.getattr("function_locations")?, + source_map: class_field(context, |c: &ModelPlanContext| &c.source_map)?, + source_warehouse_columns: class_field(context, |c: &ModelPlanContext| { + &c.source_warehouse_columns + })?, + model_locations: class_field(context, |c: &ModelPlanContext| &c.model_locations)?, + seed_locations: class_field(context, |c: &ModelPlanContext| &c.seed_locations)?, + function_locations: class_field(context, |c: &ModelPlanContext| &c.function_locations)?, }) } @@ -277,7 +281,7 @@ fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> P #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_function_sql<'py>( +pub(crate) fn resolve_function_sql<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index a614a1269d..bf84bd6ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -7,8 +7,7 @@ use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use crate::bindings::_helpers::planner_resolution::values::{ cursor_wrap, filtering_of, method, string, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::code_scanner::SqlCodeScanner; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; @@ -137,8 +136,7 @@ fn resolve_function_calls( if spans.is_empty() { return Ok(query_sql.to_owned()); } - let scanner_type = registry::native(py, NativeAttr::SqlCodeScanner)?; - let scan = scanner_type.call1((query_sql, py.None(), "SQL reference resolution"))?; + let scan = SqlCodeScanner::new(query_sql, None, "SQL reference resolution".to_owned()); let mut output = String::new(); let mut last = 0; let mut code_index = 0; @@ -146,9 +144,7 @@ fn resolve_function_calls( if table { let start = query_sql[..span.start].chars().count(); while code_index < start { - let end: Option = scan - .call_method1("non_code_end", (code_index,))? - .extract()?; + let end: Option = scan.non_code_end(py, code_index)?; code_index = end.unwrap_or(code_index + 1); } if code_index > start { @@ -175,10 +171,8 @@ fn resolve_function_calls( } else { "SQL UDF call" }; - let scanner = scanner_type.call1((query_sql, py.None(), context))?; - let end: usize = scanner - .call_method1("matching_paren", (query_sql[..suffix].chars().count(),))? - .extract()?; + let scanner = SqlCodeScanner::new(query_sql, None, context.to_owned()); + let end: usize = scanner.matching_paren(py, query_sql[..suffix].chars().count())?; let end = query_sql .char_indices() .nth(end + 1) @@ -203,9 +197,7 @@ fn resolve_function_calls( if table { let length = query_sql.chars().count(); while code_index < length { - let end: Option = scan - .call_method1("non_code_end", (code_index,))? - .extract()?; + let end: Option = scan.non_code_end(py, code_index)?; code_index = end.unwrap_or(code_index + 1); } } @@ -235,7 +227,7 @@ pub(crate) fn resolve_table_function_references( #[pyfunction] #[pyo3(signature = (*, model_locations, seed_locations, deferred_locations, selected_keys))] -fn apply_deferred_locations<'py>( +pub(crate) fn apply_deferred_locations<'py>( model_locations: &Bound<'py, PyDict>, seed_locations: &Bound<'py, PyDict>, deferred_locations: &Bound<'py, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 7e9c530b0e..4b0f31535e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -241,7 +241,7 @@ fn cast_source( #[pyfunction] #[pyo3(signature = (*, adapter, source_entry))] -fn render_source_read_relation( +pub(crate) fn render_source_read_relation( adapter: &Bound<'_, PyAny>, source_entry: &Bound<'_, PyAny>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index d22cd1591b..ea4175949b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,13 +1,18 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_hash_index; +use crate::bindings::_helpers::planner_scenario::artifacts::build_scenario_relation_map; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; +use crate::bindings::_helpers::planner_scenario::execution::build_scenario_execution_plan; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; use crate::bindings::_helpers::planner_scenario::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -38,16 +43,22 @@ pub(crate) fn build_cli_scenario_plan<'py>( let adapter = args.value("adapter")?; let (graph, warnings) = plan_scenario_graph(&scenario, &project, true, None)?; raise_for_warnings(warnings.as_any())?; - let hashes = call_with( - ®istry::native(py, NativeAttr::BuildScenarioHashIndex)?, - &[ - ("project_name", args.value("project_name")?), - ("scenarios", project.getattr("sql_scenarios")?), - ( - "run_namespace", - project.getattr("scenario")?.getattr("run_namespace")?, - ), - ], + let hashes = object( + py, + build_scenario_hash_index( + py, + Some(keywords( + py, + &[ + ("project_name", args.value("project_name")?), + ("scenarios", project.getattr("sql_scenarios")?), + ( + "run_namespace", + project.getattr("scenario")?.getattr("run_namespace")?, + ), + ], + )?), + )?, )?; let mut identities: Vec> = Vec::new(); for (field, kind) in [ @@ -70,17 +81,20 @@ pub(crate) fn build_cli_scenario_plan<'py>( )?); } } - let map = call_with( - ®istry::native(py, NativeAttr::BuildScenarioRelationMap)?, - &[ - ("scenario_name", scenario.getattr("name")?), - ("hash_prefix", hashes.get_item(scenario.getattr("name")?)?), - ("artifacts", PyTuple::new(py, identities)?.into_any()), - ( - "identifier_limit", - adapter.call_method0("maximum_identifier_length")?, - ), - ], + let map = build_scenario_relation_map( + py, + Some(keywords( + py, + &[ + ("scenario_name", scenario.getattr("name")?), + ("hash_prefix", hashes.get_item(scenario.getattr("name")?)?), + ("artifacts", PyTuple::new(py, identities)?.into_any()), + ( + "identifier_limit", + adapter.call_method0("maximum_identifier_length")?, + ), + ], + )?), )?; let (database, schema) = namespace(&project, &graph)?; let relations = build_scenario_relation_plan(&ScenarioRelations { @@ -91,19 +105,22 @@ pub(crate) fn build_cli_scenario_plan<'py>( database, schema, })?; - let result = call_with( - ®istry::native(py, NativeAttr::BuildScenarioExecutionPlan)?, - &[ - ("scenario", scenario), - ("project", project), - ("adapter", adapter), - ("graph_plan", graph), - ("relation_plan", relations), - ( - "source_lexical_syntax", - args.value("source_lexical_syntax")?, - ), - ], + let result = build_scenario_execution_plan( + py, + Some(keywords( + py, + &[ + ("scenario", scenario), + ("project", project), + ("adapter", adapter), + ("graph_plan", graph), + ("relation_plan", relations), + ( + "source_lexical_syntax", + args.value("source_lexical_syntax")?, + ), + ], + )?), )?; raise_for_warnings(&result.get_item(1)?)?; result.get_item(0) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index ac6fa6bdaf..ee04b3e7fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -1,14 +1,19 @@ //! Existing seed and function plan dataclass construction. +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_identity::metadata::build_compiled_function_fingerprint_sql; +use crate::bindings::_helpers::planner_resolution::orchestration::resolve_function_sql; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, sql::required_target, values::error, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::model_entries::extract_seed_columns; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::PyString; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; pub(super) fn seed_entries<'py>( @@ -51,10 +56,7 @@ pub(super) fn seed_entries<'py>( "file_path", seed.getattr("seed_file")?.getattr("file_path")?, ), - ( - "columns", - registry::native(py, NativeAttr::ExtractSeedColumns)?.call1((&seed,))?, - ), + ("columns", extract_seed_columns(&seed)?.into_any()), ( "csv_settings", seed.getattr("schema_entry")?.getattr("csv_settings")?, @@ -72,27 +74,30 @@ pub(super) fn function_entry<'py>( warehouse_columns: &Bound<'py, PyAny>, ) -> PyResult> { let py = function.py(); - let body = call_with( - ®istry::native(py, NativeAttr::ResolveFunctionSql)?, - &[ - ("adapter", context.adapter.clone()), - ("function", function.clone()), - ( - "model_locations", - context.relations.getattr("model_locations")?, - ), - ( - "seed_locations", - context.relations.getattr("seed_locations")?, - ), - ("function_locations", locations.clone()), - ("source_map", context.relations.getattr("source_map")?), - ("source_warehouse_columns", warehouse_columns.clone()), - ( - "star_exclude_keyword", - context.adapter.call_method0("star_exclude_keyword")?, - ), - ], + let body = resolve_function_sql( + py, + Some(keywords( + py, + &[ + ("adapter", context.adapter.clone()), + ("function", function.clone()), + ( + "model_locations", + context.relations.getattr("model_locations")?, + ), + ( + "seed_locations", + context.relations.getattr("seed_locations")?, + ), + ("function_locations", locations.clone()), + ("source_map", context.relations.getattr("source_map")?), + ("source_warehouse_columns", warehouse_columns.clone()), + ( + "star_exclude_keyword", + context.adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?), )?; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); for name in [ @@ -108,8 +113,7 @@ pub(super) fn function_entry<'py>( fields.push(("body_sql", body)); fields.push(( "fingerprint_query_sql", - registry::native(py, NativeAttr::BuildCompiledFunctionFingerprintSql)? - .call1((function,))?, + PyString::new(py, &build_compiled_function_fingerprint_sql(function)?).into_any(), )); for name in [ "fingerprint_destination", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 1303f5e260..8c52b0241b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,10 +1,18 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::compiled_project::records::compiled_model_record; +use crate::bindings::_helpers::planner_entries::model_entries::plan_model; +use crate::bindings::_helpers::planner_entries::models::ModelPlanContext; +use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; +use crate::bindings::_helpers::planner_identity::graph::topologically_order_keys; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_resolution::locations::build_function_locations; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; use crate::bindings::_helpers::planner_scenario::{ entries::{function_entry, seed_entries}, @@ -15,10 +23,10 @@ use crate::bindings::_helpers::planner_scenario::{ }; use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -75,14 +83,9 @@ pub(crate) fn build_scenario_execution_plan<'py>( }; let models = by_attribute(&project.getattr("models")?, "name")?; let functions = by_attribute(&project.getattr("functions")?, "key")?; - let locations = registry::native(py, NativeAttr::BuildFunctionLocations)? - .call1((project.getattr("functions")?,))?; - let upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((&project,))?; - let ordered = call_with( - ®istry::native(py, NativeAttr::TopologicallyOrderKeys)?, - &[("upstream", upstream)], - )?; + let locations = build_function_locations(&project.getattr("functions")?)?.into_any(); + let upstream = build_execution_upstream_deps(&project)?.into_any(); + let ordered = object(py, topologically_order_keys(&upstream, None)?)?; let mut model_keys: Vec> = Vec::new(); for key in ordered.try_iter()? { let key = key?; @@ -126,33 +129,39 @@ pub(crate) fn build_scenario_execution_plan<'py>( &model_reads::query_sql(&model)?.extract::()?, &context.relations, )?; - let record = registry::native(py, NativeAttr::CompiledModelRecord)?.call1((&model,))?; + let record = compiled_model_record(&model)?; let changes = PyDict::new(py); changes.set_item("destination", destination)?; changes.set_item("query_sql", query)?; let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; let model_context = model_context(&project, &context, (&models, &locations, &columns))?; - let result = call_with( - ®istry::native(py, NativeAttr::PlanModel)?, - &[ - ("model", model), - ("snapshot", snapshot.clone()), - ("adapter", context.adapter.clone()), - ("context", model_context), - ( - "sql_analysis_enabled", - PyBool::new(py, analysis).to_owned().into_any(), - ), - ( - "query_change_tracking", - PyBool::new(py, false).to_owned().into_any(), - ), - ("full_refresh", PyBool::new(py, true).to_owned().into_any()), - ( - "cursor_overrides", - registry::class(py, RecordId::CursorOverridePair)?.call0()?, - ), - ], + let result = object( + py, + plan_model( + py, + Some(keywords( + py, + &[ + ("model", model), + ("snapshot", snapshot.clone()), + ("adapter", context.adapter.clone()), + ("context", model_context), + ( + "sql_analysis_enabled", + PyBool::new(py, analysis).to_owned().into_any(), + ), + ( + "query_change_tracking", + PyBool::new(py, false).to_owned().into_any(), + ), + ("full_refresh", PyBool::new(py, true).to_owned().into_any()), + ( + "cursor_overrides", + registry::class(py, RecordId::CursorOverridePair)?.call0()?, + ), + ], + )?), + )?, )?; entries.push(result.get_item(0)?); for warning in result.get_item(1)?.try_iter()? { @@ -250,31 +259,30 @@ fn model_context<'py>( ) -> PyResult> { let (models, locations, columns) = inputs; let py = project.py(); - call_with( - ®istry::class(py, RecordId::ModelPlanContext)?, - &[ - ( - "model_locations", - context.relations.getattr("model_locations")?, - ), - ("models_by_name", models.clone().into_any()), - ( - "functions_by_name", - by_attribute(&project.getattr("functions")?, "name")?.into_any(), - ), - ( - "seed_locations", - context.relations.getattr("seed_locations")?, - ), - ("function_locations", locations.clone()), - ("source_map", context.relations.getattr("source_map")?), - ("source_warehouse_columns", columns.clone()), - ( - "star_exclude_keyword", - context.adapter.call_method0("star_exclude_keyword")?, - ), - ], - ) + let none = || py.None(); + Ok(Bound::new( + py, + ModelPlanContext { + model_locations: context.relations.getattr("model_locations")?.unbind(), + models_by_name: models.clone().into_any().unbind(), + functions_by_name: by_attribute(&project.getattr("functions")?, "name")? + .into_any() + .unbind(), + seed_locations: context.relations.getattr("seed_locations")?.unbind(), + function_locations: locations.clone().unbind(), + source_map: context.relations.getattr("source_map")?.unbind(), + source_warehouse_columns: columns.clone().unbind(), + star_exclude_keyword: context + .adapter + .call_method0("star_exclude_keyword")? + .unbind(), + runtime_cursor_producer_names: PyFrozenSet::empty(py)?.into_any().unbind(), + future_cursor_config: none(), + start_cursor_config: none(), + invocation_time: none(), + }, + )? + .into_any()) } fn expectations<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs index c928d53cfa..66d16970e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod artifacts; mod cli; mod constants; mod entries; -mod execution; +pub(crate) mod execution; mod fixtures; mod graph; mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index 32528173d4..797ed05898 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -1,6 +1,7 @@ //! Resolve scenario SQL through the shared native scanner and adapter callbacks. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, }; @@ -8,8 +9,7 @@ use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, values::{error, string}, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyStringMethods}; @@ -46,7 +46,7 @@ pub(super) fn resolve_sql( result = replace_markers(&result, kind, Some(syntax.clone().into()), |name, marker| { let target = locations.call_method1("get", (name,))?; let target = if target.is_none() {target} else if kind == super::constants::SOURCE_MARKER { - call_with(®istry::native(py, NativeAttr::RenderSourceRelation)?, &[("entry", target), ("adapter", context.adapter.clone())])? + object(py, render_source_relation(&target, Some(&context.adapter.clone()).filter(|value| !value.is_none()))?)? } else {target.getattr("qualified_name")?}; if target.is_none() { let marker = string(py, marker).repr()?.to_str()?.to_owned(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs index e7194181c9..3ad6d7b82e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -1,7 +1,7 @@ //! Planner scope Python boundary models. -mod loaders; +pub(crate) mod loaders; pub(crate) mod registration; -mod scope; -mod sources; +pub(crate) mod scope; +pub(crate) mod sources; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index a06abf06ca..769f3ff02d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -119,7 +119,9 @@ fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult(raw: &Bound<'py, PyString>) -> PyResult> { +pub(crate) fn parse_project_selector<'py>( + raw: &Bound<'py, PyString>, +) -> PyResult> { let carrier: Carrier<'py> = Carrier::single(raw)?; let parsed: ParsedSelector = parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; @@ -129,7 +131,7 @@ fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult( +pub(crate) fn resolve_graph_selection<'py>( graph: PyRef<'py, NativeProjectGraph>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, @@ -140,7 +142,7 @@ fn resolve_graph_selection<'py>( /// `--select` minus `--exclude` on a project graph, adding the functions selected models need. #[pyfunction] #[pyo3(signature = (*, graph, select, exclude))] -fn select_project_graph<'py>( +pub(crate) fn select_project_graph<'py>( graph: &Bound<'py, PyAny>, select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, @@ -234,7 +236,7 @@ fn match_project_selector<'py>( /// The upstream key closure of one compiled graph key. #[pyfunction] #[pyo3(signature = (*, key, upstream_deps))] -fn expand_project_upstream_keys<'py>( +pub(crate) fn expand_project_upstream_keys<'py>( key: Bound<'py, PyAny>, upstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index b420ee0868..d0c203f978 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -164,7 +164,7 @@ fn split_unit_test_selectors<'py>( /// Raise when a selector names a unit test; return the unchanged resource selectors. #[pyfunction] #[pyo3(signature = (*, select, exclude, discovered_inputs))] -fn reject_unit_test_selectors<'py>( +pub(crate) fn reject_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 4e5f95a646..7f917b0b88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -1,5 +1,10 @@ //! Assemble audit plan dataclasses using native rendering and scheduling. +use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; +use crate::bindings::_helpers::auditing::render::render_audit_sql; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::graph::views::project_lineage_views; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::audits::{ reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, }; @@ -16,7 +21,7 @@ use crate::bindings::_helpers::planner_strategy::constants::{ AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, }; use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Render and validate one audit query with the supplied adapter callbacks. @@ -26,22 +31,22 @@ fn render_query<'py>( context: &str, ) -> PyResult> { let py = sql.py(); - let resolved = call_with( - ®istry::native(py, NativeAttr::RenderAuditSql)?, - &[ - ("unresolved_sql", sql), - ("model_locations", request.model_locations.clone()), - ("seed_locations", request.seed_locations.clone()), - ("source_map", request.source_map.clone()), - ("adapter", request.adapter.clone()), - ], + let resolved = object( + py, + render_audit_sql(Some(&keywords( + py, + &[ + ("unresolved_sql", sql), + ("model_locations", request.model_locations.clone()), + ("seed_locations", request.seed_locations.clone()), + ("source_map", request.source_map.clone()), + ("adapter", request.adapter.clone()), + ], + )?))?, )?; - call_with( - ®istry::native(py, NativeAttr::AssertNoUnresolvedSqlMarkers)?, - &[ - ("sql", resolved.clone()), - ("context", PyString::new(py, context).into_any()), - ], + object( + py, + assert_no_unresolved_sql_markers(py, resolved.downcast::()?, context)?, )?; Ok(resolved) } @@ -230,7 +235,7 @@ pub(crate) fn build_selected_audit_entries<'py>( if selected.is_empty() { return Ok(entries); } - let lineage = registry::native(py, NativeAttr::ProjectLineageViews)?.call1((project,))?; + let lineage = project_lineage_views(py, &project)?; let relations = request.get("relations")?; let planning = AuditPlanning { model_locations: relations.getattr("model_locations")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index ff00ec515e..faa8aca874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -44,7 +44,7 @@ fn log_parse_failure(py: Python<'_>, failure: &TypeParseFailure) -> PyResult<()> /// The `CursorType` a supported cross-adapter SQL type holds, or `None`. #[pyfunction] #[pyo3(signature = (sql_type))] -fn classify_cursor_sql_type<'py>( +pub(crate) fn classify_cursor_sql_type<'py>( py: Python<'py>, sql_type: &str, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 7a9b13e5d2..562eb830b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::executor::ExecutorHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; pub(crate) const RELATIONS_MAIN_LIST: AdapterHost = AdapterHost::ListRelationsForInspection; pub(crate) const RELATIONS_MAIN_COLUMNS: AdapterHost = AdapterHost::GetColumnsForInspection; @@ -14,10 +14,6 @@ pub(crate) const QUERY_RECORD: AdapterHost = AdapterHost::InspectionQueryRecord; pub(crate) const COLUMN_INFO: AdapterHost = AdapterHost::ColumnInfo; pub(crate) const RELATION_LOOKUP: AdapterHost = AdapterHost::RelationLookup; pub(crate) const LOG_DEBUG_EVENT: ExecutorHost = ExecutorHost::LogDebugEvent; -pub(crate) const CURSOR_POLICIES: NativeAttr = NativeAttr::ResolveStartCursorConfig; -pub(crate) const MAXIMUM_START: NativeAttr = NativeAttr::MaximumAllowedStart; -pub(crate) const RENDER_SOURCE_READ: NativeAttr = NativeAttr::RenderSourceReadRelation; -pub(crate) const YAML_ENTRY_LINE: NativeAttr = NativeAttr::YamlEntryLine; pub(crate) const RUN_SCOPE: ExecutorHost = ExecutorHost::RunScope; pub(crate) const SQL_HOOK_ENTRY: RecordId = RecordId::SqlHookEntry; pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 5fff2d990a..e8e8b9cc21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,23 +1,36 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::cursor::intervals::partition; +use crate::bindings::_helpers::cursor::scalars; +use crate::bindings::_helpers::cursor::scalars::compare_values; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::cursor::scalars::to_inclusive; +use crate::bindings::_helpers::model_config::config_values::config_cursor_bound; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ - by_name, config_str, model_key, native, native_member, optional_text, planner_error, - planner_model, text, tuple, + by_name, config_str, model_key, native_member, optional_text, planner_error, planner_model, + text, tuple, }; +use crate::bindings::_helpers::planner_resolution::cursor_policies::resolve_start_cursor_config; +use crate::bindings::_helpers::planner_resolution::cursor_windows::normalize_cursor_snapshot_grain; +use crate::bindings::_helpers::planner_resolution::cursor_windows::resolve_effective_timestamp_grain; +use crate::bindings::_helpers::planner_resolution::maximum_safety::maximum_allowed_start; +use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; use crate::bindings::_helpers::planner_warehouse::constants::{ - CURSOR_ALGEBRA_ERROR, CURSOR_POLICIES, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, - LOG_DEBUG_EVENT, MAXIMUM_START, PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, - RUN_RECORDED_QUERY, + CURSOR_ALGEBRA_ERROR, CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, DATE_VALUE, LOG_DEBUG_EVENT, + PLANNER_LOGGER, QUERY_RECORD, RECORD_QUERY, RUN_BOUNDED, RUN_RECORDED_QUERY, }; use crate::bindings::_helpers::planner_warehouse::snapshot::{ SnapshotRequest, inspection_concurrency, }; use crate::bindings::_helpers::planner_warehouse::watermarks::validate_watermark_column; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -96,19 +109,11 @@ fn parse<'py>( raw: &Bound<'py, PyAny>, cursor_type: &str, ) -> PyResult> { - native( - py, - NativeAttr::CursorParse, - &[("raw", raw.clone()), ("cursor_type", text(py, cursor_type))], - ) + scalars::parse(raw, &text(py, cursor_type)) } fn render<'py>(value: &Bound<'py, PyAny>) -> PyResult> { - native( - value.py(), - NativeAttr::CursorRender, - &[("value", value.clone())], - ) + object(value.py(), render_value(value)?) } fn grain<'py>(py: Python<'py>, value: &str) -> PyResult> { @@ -238,13 +243,12 @@ fn upstream_relation<'py>( } else if kind.eq("source")? && let Some(source) = maps.sources.get_item(&name)? { - return native( + return object( py, - NativeAttr::RenderSourceRelation, - &[ - ("entry", source.getattr("source_entry")?), - ("adapter", adapter.clone()), - ], + render_source_relation( + &source.getattr("source_entry")?, + Some(adapter).filter(|value| !value.is_none()), + )?, ); } Ok(py.None().into_bound(py)) @@ -255,15 +259,11 @@ fn lineage_reference<'py>( input_name: &Bound<'py, PyAny>, maps: &ModelMaps<'py>, ) -> PyResult> { - native( - model.py(), - NativeAttr::ResolveLineageReference, - &[ - ("model", model.clone()), - ("input_name", input_name.clone()), - ("models_by_name", maps.models.clone().into_any()), - ("functions_by_name", maps.functions.clone().into_any()), - ], + resolve_lineage_reference( + model, + &input_name.extract::()?, + maps.models.clone().into_any(), + maps.functions.clone().into_any(), ) } @@ -279,13 +279,9 @@ fn terminal_bound<'py>( let Some(upstream) = maps.models.get_item(reference.getattr("ref_name")?)? else { return Ok(None); }; - let value = native( + let value = object( py, - NativeAttr::GetConfigCursorBound, - &[ - ("values", upstream.getattr("config")?.getattr("values")?), - ("key", text(py, key)), - ], + config_cursor_bound(&upstream.getattr("config")?.getattr("values")?, key)?, )?; if value.is_none() { return Ok(None); @@ -361,12 +357,7 @@ fn collect_models<'py>( else { continue; }; - let watermark_inputs = native( - py, - NativeAttr::ResolveCursorInputRoles, - &[("model", model.clone())], - )? - .getattr("watermark_inputs")?; + let watermark_inputs = resolve_cursor_input_roles(&model)?.getattr("watermark_inputs")?; let watermark_inputs: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = watermark_inputs .call_method0("items")? .try_iter()? @@ -460,20 +451,13 @@ fn collect_models<'py>( None => py.None().into_bound(py), }); } - let effective_grain = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveTimestampGrain)?, - &[ - ("cursor_type", optional_text(py, cursor_type.as_deref())), - ( - "downstream_grain", - optional_text(py, cursor_grain.as_deref()), - ), - ( - "microbatch_strategy", - optional_text(py, strategy.as_deref()), - ), - ("cursor_input_grains", tuple(py, input_grains)?), - ], + let effective_grain = resolve_effective_timestamp_grain( + &optional_text(py, cursor_type.as_deref()), + &optional_text(py, cursor_grain.as_deref()), + &tuple(py, input_grains)?, + optional_text(py, strategy.as_deref()) + .extract::>()? + .as_deref(), )?; let mode = config_str(&model, "cursor_watermark_mode")? .filter(|value| !value.is_empty()) @@ -492,13 +476,7 @@ fn collect_models<'py>( cursor_column, upstreams, effective_grain, - start_cursor_config: call_with( - ®istry::native(py, CURSOR_POLICIES)?, - &[ - ("model", model.clone()), - ("project_config", inputs.start_cursor_config.clone()), - ], - )?, + start_cursor_config: resolve_start_cursor_config(&model, &inputs.start_cursor_config)?, has_start_override: start_override_set(&model, &inputs.cursor_overrides)?, watermark_all: watermark_mode.eq("all")?, watermark_mode, @@ -851,22 +829,23 @@ fn eligible_target_maxes<'py>( { continue; } - let allowed = call_with( - ®istry::native(py, MAXIMUM_START)?, - &[ - ("discovered_value", render(target_max)?), - ("cursor_grain", info.effective_grain.clone()), - ("invocation_time", invocation_time.clone()), - ("max_ahead", config.getattr("max_ahead")?), - ], - )?; - let horizon = parse(py, &allowed, CURSOR_TYPE_TIMESTAMP)?; - let order: i64 = native( + let allowed = text( py, - NativeAttr::CursorCompare, - &[("left", target_max.clone()), ("right", horizon.clone())], - )? - .extract()?; + &maximum_allowed_start( + py, + Some(keywords( + py, + &[ + ("discovered_value", render(target_max)?), + ("cursor_grain", info.effective_grain.clone()), + ("invocation_time", invocation_time.clone()), + ("max_ahead", config.getattr("max_ahead")?), + ], + )?), + )?, + ); + let horizon = parse(py, &allowed, CURSOR_TYPE_TIMESTAMP)?; + let order: i64 = object(py, compare_values(target_max, &horizon)?)?.extract()?; if order <= 0 { continue; } @@ -935,28 +914,19 @@ fn eligible_target_maxes<'py>( .filter(|value| !value.is_empty()) .unwrap_or(CURSOR_TYPE_TIMESTAMP); let value = parse(py, &first, cursor_type)?; - let normalized = call_with( - ®istry::native(py, NativeAttr::NormalizeCursorSnapshotGrain)?, - &[ - ( - "cursor_snapshot", - planner_model( - py, - RecordId::ModelCursorSnapshot, - &[ - ("target_max", py.None().into_bound(py)), - ("upstream_mins", PyTuple::empty(py).into_any()), - ("upstream_maxes", PyTuple::empty(py).into_any()), - ("target_eligible_max", value.clone()), - ], - )?, - ), - ( - "cursor_type", - optional_text(py, info.cursor_type.as_deref()), - ), - ("effective_grain", info.effective_grain.clone()), - ], + let normalized = normalize_cursor_snapshot_grain( + &planner_model( + py, + RecordId::ModelCursorSnapshot, + &[ + ("target_max", py.None().into_bound(py)), + ("upstream_mins", PyTuple::empty(py).into_any()), + ("upstream_maxes", PyTuple::empty(py).into_any()), + ("target_eligible_max", value.clone()), + ], + )?, + &optional_text(py, info.cursor_type.as_deref()), + &info.effective_grain.clone(), )? .getattr("target_eligible_max")?; values.insert( @@ -1030,14 +1000,7 @@ fn assemble<'py>( } } let effective_max = match &upstream.terminal_end { - Some(end) => Some(native( - py, - NativeAttr::CursorExclusiveToInclusive, - &[ - ("value", end.clone()), - ("grain", cursor_grain(py, info, upstream)?), - ], - )?), + Some(end) => Some(to_inclusive(end, &cursor_grain(py, info, upstream)?)?), None => max_value.cloned(), }; if let Some(min_value) = min_value { @@ -1090,15 +1053,7 @@ fn assemble<'py>( && let Some(max_value) = max_value { available = Some( - native( - py, - NativeAttr::CursorObservedPartition, - &[ - ("value", max_value.clone()), - ("grain", cursor_grain(py, info, upstream)?), - ], - )? - .getattr("end")?, + partition(max_value, &cursor_grain(py, info, upstream)?)?.getattr("end")?, ); } if let Some(available) = available.filter(|value| !value.is_none()) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 7fe85474b5..66de58712c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -3,7 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_phases::objects::{native, optional_text, text}; +use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; +use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; +use crate::bindings::_helpers::planner_phases::objects::object; +use crate::bindings::_helpers::planner_phases::objects::{optional_text, text}; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::exceptions::{PyOSError, PyValueError}; @@ -276,20 +279,14 @@ impl MigrationFingerprintCache { let metadata_json: String = args.value("metadata_json")?.extract()?; let ref_identities = &args.value("ref_identities")?; let dialect: Option = args.value("dialect")?.extract()?; - let metadata_json: String = registry::native(py, NativeAttr::WithoutDeclaredColumnsHash)? - .call1((metadata_json,))? - .extract()?; + let metadata_json: String = without_declared_columns_hash(&metadata_json); let definition = (query_sql.clone(), metadata_json.clone()); let names = match self.ref_names.get(&definition) { Some(names) => names.clone(), None => { - let found = native( + let found = object( py, - NativeAttr::MigrationFingerprintRefNames, - &[ - ("query_sql", text(py, &query_sql)), - ("metadata_json", text(py, &metadata_json)), - ], + migration_fingerprint_ref_names(py, &query_sql, &metadata_json)?, )?; let mut names: Vec = found .try_iter()? @@ -319,9 +316,8 @@ impl MigrationFingerprintCache { for (name, identity) in &key.3 { pyo3::types::PyDictMethods::set_item(&identities, name, identity)?; } - let computed: Option = native( - py, - NativeAttr::BuildMigrationFingerprint, + let computed: Option = call_with( + ®istry::native(py, NativeAttr::BuildMigrationFingerprint)?, &[ ("query_sql", text(py, &key.0)), ("metadata_json", text(py, &key.1)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index d0a9af43c6..b6859b1eec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -3,18 +3,24 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::fingerprints::latest_rows::select_latest_fingerprints; +use crate::bindings::_helpers::fingerprints::storage_io::read_latest_fingerprints; +use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_identity::graph::build_execution_upstream_deps; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ by_name, frozenset, model_key, native, text, tuple, }; +use crate::bindings::_helpers::planner_scope::loaders::build_upstream_intermediate_source_map; use crate::bindings::_helpers::planner_warehouse::constants::{ ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -84,19 +90,8 @@ fn relevant_state_keys<'py>( if selected.is_none() { return Ok(selected.clone()); } - let deps = registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((project,))?; - native( - py, - NativeAttr::TransitiveClosureMany, - &[ - ("starts", selected.clone()), - ("edges", deps), - ( - "include_starts", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], - ) + let deps = build_execution_upstream_deps(project)?.into_any(); + object(py, transitive_closure_many(selected, &deps, true, None)?) } fn resolve_database<'py>( @@ -210,13 +205,9 @@ fn metadata_name_filter( names.insert(source_table_name(&entry)?); } } - let intermediate = native( + let intermediate = object( py, - NativeAttr::BuildUpstreamIntermediateSourceMap, - &[ - ("project", project.clone()), - ("selected_keys", selected.clone()), - ], + build_upstream_intermediate_source_map(project, selected)?, )?; for entry in intermediate.call_method0("values")?.try_iter()? { names.insert(source_table_name(&entry?)?); @@ -522,9 +513,8 @@ fn gather_fingerprints<'py>( )?; for schema in schemas { let schema_lower = lower(&text(py, schema))?; - let mut set = native( + let mut set = read_latest_fingerprints(Some(&keywords( py, - NativeAttr::ReadLatestFingerprints, &[ ("connection", request.connection.clone()), ("execute", request.execute.clone()), @@ -558,7 +548,7 @@ fn gather_fingerprints<'py>( ), ("filtered_node_types", filtered_types.clone()), ], - )?; + )?))?; if read_unfiltered { let by_identity = set.getattr("fingerprints_by_identity")?; let values: Vec> = if by_identity.is_none() { @@ -571,14 +561,11 @@ fn gather_fingerprints<'py>( }; unfiltered.set_item(&schema_lower, tuple(py, values)?)?; if node_names.is_some() { - set = native( + set = select_latest_fingerprints( py, - NativeAttr::SelectLatestFingerprints, - &[ - ("fingerprint_set", set), - ("node_names", node_names_value.clone()), - ("filtered_node_types", filtered_types.clone()), - ], + &set, + node_names_value.extract::>()?, + filtered_types.extract::>()?, )?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index b55cad2956..c52a74fe9d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -1,16 +1,18 @@ //! Source read deferral for managed source loaders. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ - native, optional_text, planner_error, replace, text, + optional_text, planner_error, replace, text, }; use crate::bindings::_helpers::planner_warehouse::constants::{ MODEL_HOOK_CONFIG_KEYS, SQL_HOOK_ENTRY, }; -use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -128,14 +130,11 @@ fn resolve_target<'py>( local_config: &Bound<'py, PyAny>, target_name: Bound<'py, PyAny>, ) -> PyResult> { - native( + resolve_target_config( project_config.py(), - NativeAttr::ResolveTargetConfig, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_name", target_name), - ], + project_config, + local_config, + &target_name, ) } @@ -192,16 +191,13 @@ fn target_field<'py>( return Ok(None); } Ok(Some( - native( + object( py, - NativeAttr::RenderProjectVarText, - &[ - ("value", effective_vars.get_item(name)?), - ( - "label", - text(py, &format!("source deferral variable '${{{name}}}'")), - ), - ], + render_project_var_text( + &effective_vars.get_item(name)?, + &text(py, &format!("source deferral variable '${{{name}}}'")) + .extract::()?, + )?, )? .extract()?, )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs index ea54c1ccd9..1dc2ffc0d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_tables.rs @@ -2,17 +2,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::discovery::locations::yaml_entry_line; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{planner_error, tuple}; use crate::bindings::_helpers::planner_warehouse::constants::{ - MISSING_SOURCE_TABLE_CODE, RENDER_SOURCE_READ, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, - YAML_ENTRY_LINE, + MISSING_SOURCE_TABLE_CODE, RUN_BOUNDED, WILDCARD_TABLE_CHARACTER, }; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::references::source_relations::render_source_relation; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; +use pyo3::types::PyString; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_help::missing_source_tables_help; use sqlbuild_analysis::planner_warehouse::main::missing_source_tables_message::missing_source_tables_message; @@ -66,13 +67,7 @@ fn read_relation<'py>( adapter: &Bound<'py, PyAny>, entry: &Bound<'py, PyAny>, ) -> PyResult> { - call_with( - ®istry::native(adapter.py(), RENDER_SOURCE_READ)?, - &[ - ("adapter", adapter.clone()), - ("source_entry", entry.clone()), - ], - ) + Ok(PyString::new(adapter.py(), &render_source_relation(entry, Some(adapter))?).into_any()) } fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { @@ -81,18 +76,14 @@ fn declaration_location(source: &Bound<'_, PyAny>) -> PyResult { .getattr("relative_path")? .call_method0("as_posix")? .extract()?; - let line = call_with( - ®istry::native(source.py(), YAML_ENTRY_LINE)?, - &[ - ("contents", file.getattr("contents")?), - ("name", source.getattr("name")?), - ], - )?; - if line.is_none() { - Ok(path) - } else { - Ok(format!("{path}:{}", line.str()?)) - } + let line = yaml_entry_line( + &file.getattr("contents")?.extract::()?, + &source.getattr("name")?.extract::()?, + ); + Ok(match line { + None => path, + Some(line) => format!("{path}:{line}"), + }) } /// Raise S405 when a selected resource reads an external source table that does not exist. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index f9c629d6fb..879911322b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,12 +1,12 @@ //! Validate reliable watermark column contracts before warehouse inspection. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; +use crate::bindings::_helpers::planner_strategy::cursor_types::classify_cursor_sql_type; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, }; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods}; @@ -24,8 +24,10 @@ pub(crate) fn watermark_type_is_compatible( else { return Ok(true); }; - let classified = registry::native(declared_type.py(), NativeAttr::ClassifyCursorSqlType)? - .call1((declared_type,))?; + let classified = object( + declared_type.py(), + classify_cursor_sql_type(declared_type.py(), &declared_type.extract::()?)?, + )?; classified.eq(expected) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 4926d5bdc0..c6574dadcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -320,7 +320,6 @@ native_ids! { MicrobatchStrategy => "MicrobatchStrategy", MicrobatchesConfig => "MicrobatchesConfig", MigrationCompatibility => "MigrationCompatibility", - MigrationCompatibilityResult => "MigrationCompatibilityResult", MigrationDecision => "MigrationDecision", MigrationDiscovery => "MigrationDiscovery", MigrationEvent => "MigrationEvent", @@ -333,11 +332,7 @@ native_ids! { ModelColumnLineage => "ModelColumnLineage", ModelCursorSnapshot => "ModelCursorSnapshot", ModelInputBuildContext => "ModelInputBuildContext", - ModelMigrationDeclaration => "ModelMigrationDeclaration", - ModelMigrationDiscovery => "ModelMigrationDiscovery", ModelMigrationPlanEntry => "ModelMigrationPlanEntry", - ModelMigrationRequest => "ModelMigrationRequest", - ModelPlanContext => "ModelPlanContext", ModelPlanEntry => "ModelPlanEntry", ModelSchemaDeclaration => "ModelSchemaDeclaration", ModelSql => "ModelSql", @@ -381,7 +376,6 @@ native_ids! { PathSelector => "PathSelector", PlanAction => "PlanAction", PlanCommandRequest => "PlanCommandRequest", - PlanEntryBuildInputs => "PlanEntryBuildInputs", PlanOutput => "PlanOutput", PlanOutputExtras => "PlanOutputExtras", PlanProviderUsage => "PlanProviderUsage", @@ -390,13 +384,10 @@ native_ids! { PlanWarning => "PlanWarning", PlannedSqlTests => "PlannedSqlTests", PlannerChangeResults => "PlannerChangeResults", - PlannerIdentityContext => "PlannerIdentityContext", PlannerOverrides => "PlannerOverrides", PlannerPolicies => "PlannerPolicies", - PlannerRelationsContext => "PlannerRelationsContext", PlannerResolvedActions => "PlannerResolvedActions", PlannerScope => "PlannerScope", - PlannerScopeResolution => "PlannerScopeResolution", PlannerSelection => "PlannerSelection", PlaygroundCommandRequest => "PlaygroundCommandRequest", PlaygroundTemplate => "PlaygroundTemplate", @@ -766,7 +757,6 @@ native_ids! { CheckCompileReuse => "check_compile_reuse", CheckCursorTypeConsistency => "check_cursor_type_consistency", CheckFullRebuildAllowed => "check_full_rebuild_allowed", - CheckMigrationCompatibility => "check_migration_compatibility", CheckSelectedScopeBuildability => "check_selected_scope_buildability", ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", @@ -812,7 +802,6 @@ native_ids! { DbtInitProgressReporter => "DbtInitProgressReporter", DbtRunner => "DbtRunner", DebugDbtFromProject => "debug_dbt_from_project", - DecideModelMigration => "decide_model_migration", DeclarationRows => "declaration_rows", DeclaredColumnAliases => "declared_column_aliases", DeclaredRelations => "declared_relations", @@ -827,7 +816,6 @@ native_ids! { DiscoverHookFunctions => "discover_hook_functions", DiscoverMaterializationFiles => "discover_materialization_files", DiscoverModelFiles => "discover_model_files", - DiscoverModelMigrations => "discover_model_migrations", DiscoverModelSchemaFiles => "discover_model_schema_files", DiscoverProjectInputs => "discover_project_inputs", DiscoverProviderClasses => "discover_provider_classes", @@ -933,24 +921,14 @@ native_ids! { MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", - MatchMigrationFingerprints => "match_migration_fingerprints", MatchProjectRelation => "match_project_relation", MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", MergeRecomputedModelChanges => "merge_recomputed_model_changes", MicrobatchGuidance => "microbatch_guidance", - MigrationDestinationFingerprints => "migration_destination_fingerprints", MigrationFingerprintCache => "MigrationFingerprintCache", MigrationFingerprintRefNames => "migration_fingerprint_ref_names", - MigrationMissingOriginPolicy => "migration_missing_origin_policy", - MigrationPlanningDatabase => "migration_planning_database", - MigrationProjectSchemas => "migration_project_schemas", - MigrationRedirectedCursorSnapshots => "migration_redirected_cursor_snapshots", - MigrationRequireSameDatabase => "migration_require_same_database", - MigrationResolveOrigin => "migration_resolve_origin", - MigrationStateInspection => "MigrationStateInspection", - MigrationWithExecution => "migration_with_execution", MissingUpstreamMessage => "missing_upstream_message", ModelDefinitionHash => "model_definition_hash", ModelHeaderFailure => "model_header_failure", @@ -1158,7 +1136,6 @@ native_ids! { StandalonePivotModels => "standalone_pivot_models", StartModelAnalysisSession => "start_model_analysis_session", StaticSqlTestPlanningDiagnostics => "static_sql_test_planning_diagnostics", - StoredMigrationFingerprint => "stored_migration_fingerprint", StoredOldNameViewColumns => "stored_old_name_view_columns", TerminalEventIndex => "TerminalEventIndex", TerminalEventIndexContext => "_terminal_event_index_context", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs index 721d165ef3..a46a8dc240 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod registration; -mod relations; +pub(crate) mod relations; pub(crate) mod source_relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs index 4ef70d6c7a..de66cb5576 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs @@ -70,7 +70,7 @@ fn parts_compatible(left: Option<&str>, right: Option<&str>) -> bool { /// The project relation `relation` names, resolving missing qualifiers from the defaults. #[pyfunction] #[pyo3(signature = (*, index, relation, default_database = None, default_schema = None))] -fn match_project_relation<'py>( +pub(crate) fn match_project_relation<'py>( index: &Bound<'py, PyAny>, relation: &Bound<'py, PyAny>, default_database: Option, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 836e7da804..074c9f7beb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -785,7 +785,7 @@ fn degraded_document( /// Write execution JSON to stdout or a requested side-channel file. #[pyfunction] #[pyo3(signature = (*, payload, json_output, json_output_path))] -fn write_execution_json_output( +pub(crate) fn write_execution_json_output( py: Python<'_>, payload: Bound<'_, PyAny>, json_output: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs index df2a6ce8d4..78d947b28b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/terminal_index.rs @@ -287,7 +287,7 @@ impl TerminalEventClaim { /// Return the terminal event index installed for this invocation. #[pyfunction] -fn current_terminal_event_index(py: Python<'_>) -> PyResult> { +pub(crate) fn current_terminal_event_index(py: Python<'_>) -> PyResult> { context_variable(py)?.call_method0("get") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index ca768df886..f6afb5970a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -36,7 +36,7 @@ type Object<'py> = Bound<'py, PyAny>; /// `source_binding_diagnostics(*, project, columns, profile, selected_keys)`. #[pyfunction] #[pyo3(signature = (**request))] -fn source_binding_diagnostics<'py>( +pub(crate) fn source_binding_diagnostics<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("source_binding_diagnostics", request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index d10b065dd6..8a51d31475 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -4,11 +4,13 @@ use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timi use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; +use crate::bindings::_helpers::sql_test_glue::planning::resolve_compiled_sql_test_chains; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -33,7 +35,10 @@ pub(super) fn chains<'py>( ), ], )?; - let native = registry::native(py, NativeAttr::ResolveCompiledSqlTestChains)?.call1((request,)); + let native = request + .extract() + .and_then(|request| resolve_compiled_sql_test_chains(py, request)) + .and_then(|chains| object(py, chains)); let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index e7bf16eaf4..35c35da692 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -48,7 +48,7 @@ type PlanRow = ( /// The Python planning request: compiled models and tests, with the adapter's facts. #[derive(FromPyObject)] -struct PlanningRequest<'py> { +pub(crate) struct PlanningRequest<'py> { #[pyo3(attribute)] models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. @@ -79,7 +79,7 @@ struct PlanningRequest<'py> { /// The Python chain request: compiled models and the tests whose chains to resolve. #[derive(FromPyObject)] -struct ChainRequest<'py> { +pub(crate) struct ChainRequest<'py> { #[pyo3(attribute)] models: Vec>, #[pyo3(attribute)] @@ -90,7 +90,7 @@ struct ChainRequest<'py> { /// Each test's ordered unmocked model chain; direct-logic tests have none. #[pyfunction] -fn resolve_compiled_sql_test_chains( +pub(crate) fn resolve_compiled_sql_test_chains( py: Python<'_>, request: ChainRequest<'_>, ) -> PyResult>> { @@ -118,7 +118,7 @@ fn resolve_compiled_sql_test_chains( /// Plan SQL tests in one batch and return each plan with its distinct error messages. #[pyfunction] -fn plan_compiled_sql_tests( +pub(crate) fn plan_compiled_sql_tests( py: Python<'_>, request: PlanningRequest<'_>, ) -> PyResult<(Vec, u64, u64)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs index bbcc55a893..22ff949e69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -25,7 +25,7 @@ pub(crate) struct SqlCodeScanner { impl SqlCodeScanner { #[new] #[pyo3(signature = (sql, syntax, context))] - fn new(sql: &str, syntax: Option, context: String) -> Self { + pub(crate) fn new(sql: &str, syntax: Option, context: String) -> Self { Self { scanner: code_scanner(sql, syntax.map(Into::into)), context, @@ -33,12 +33,12 @@ impl SqlCodeScanner { } /// The end of the comment or quoted text at `index`, or `None` when code starts there. - fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { + pub(crate) fn non_code_end(&self, py: Python<'_>, index: usize) -> PyResult> { scanner_non_code_end(&self.scanner, index).map_err(|unclosed| self.error(py, unclosed)) } /// The index of the parenthesis closing the one at `open`. - fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { + pub(crate) fn matching_paren(&self, py: Python<'_>, open: usize) -> PyResult { scanner_matching_paren(&self.scanner, open).map_err(|unclosed| self.error(py, unclosed)) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs index 4116afb30d..8cfb088388 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -222,7 +222,9 @@ fn read_state_events<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn read_histories<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("read_old_name_view_histories", request)?; let access: StateAccess<'py> = access(&request)?; let location = || -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -283,7 +285,7 @@ fn read_renames<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( +pub(crate) fn stored_columns_binding<'py>( adapter: &Bound<'py, PyAny>, connection: &Bound<'py, PyAny>, database: Bound<'py, PyAny>, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4504f51c5a..3d44ad436d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -256,32 +256,6 @@ def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[ STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] -def migration_planning_database(*, runtime: PlannerRuntime) -> str | None: ... -def migration_project_schemas(*, runtime: PlannerRuntime) -> set[str]: ... -def migration_missing_origin_policy(*, runtime: PlannerRuntime) -> MissingMigrationOriginPolicy: ... -def migration_require_same_database( - *, state: MigrationStateInspection, location: CompiledRelationLocation -) -> None: ... -def migration_resolve_origin( - *, request: ModelMigrationRequest, runtime: PlannerRuntime, state: MigrationStateInspection -) -> tuple[CompiledRelationLocation, str | None]: ... -def migration_promotion( - *, runtime: PlannerRuntime, decision: MigrationDecision -) -> MigrationPromotion: ... -def migration_with_execution( - *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry -) -> ModelMigrationPlanEntry: ... -def migration_redirected_cursor_snapshots( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - existing_relations: dict[str, RelationInfo], - target_relations: dict[str, str], - overrides: PlannerOverrides, - deferral: DeferralInputs, - origin_cursor_columns: dict[str, str] | None = None, -) -> dict[str, ModelCursorSnapshot]: ... - class CompileInputReads: """Environment, settings, and run-identity reads observed during one compile.""" @@ -980,7 +954,7 @@ class UnitTestSelectorSplit(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> UnitTestSelectorSplit: ... -class ModelPlanContext(NativeRecord): +class ModelPlanContext: """Relation lookups and source columns for building one model plan entry.""" model_locations: dict[str, CompiledRelationLocation] @@ -997,6 +971,7 @@ class ModelPlanContext(NativeRecord): invocation_time: datetime | None def __init__( self, + *, model_locations: dict[str, CompiledRelationLocation], models_by_name: dict[str, CompiledModelView], seed_locations: dict[str, CompiledRelationLocation], @@ -1010,7 +985,6 @@ class ModelPlanContext(NativeRecord): start_cursor_config: StartCursorsConfig | None = ..., invocation_time: datetime | None = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> ModelPlanContext: ... class CursorOverridePair(NativeRecord): """Resolved per-model cursor start/end overrides for plan entry construction.""" @@ -1022,7 +996,7 @@ class CursorOverridePair(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> CursorOverridePair: ... -class PlanEntryBuildInputs(NativeRecord): +class PlanEntryBuildInputs: """Blocked models and cursor overrides for plan entry building.""" source_freshness_blocked_model_names: frozenset[str] @@ -1037,6 +1011,7 @@ class PlanEntryBuildInputs(NativeRecord): microbatch_limit_action: MicrobatchLimitAction def __init__( self, + *, source_freshness_blocked_model_names: frozenset[str] = ..., external_blocked_model_names: frozenset[str] = ..., start_cursor_override: str | None = ..., @@ -1048,7 +1023,6 @@ class PlanEntryBuildInputs(NativeRecord): max_microbatches_is_override: bool = ..., microbatch_limit_action: MicrobatchLimitAction = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlanEntryBuildInputs: ... class FunctionChangeResult(NativeRecord): """Per-function output from change detection.""" @@ -1340,7 +1314,7 @@ class PlannerRuntime: invocation_time: datetime = ..., ) -> None: ... -class PlannerScopeResolution(NativeRecord): +class PlannerScopeResolution: """Resolved planner scopes for one plan build.""" selected_scope: PlannerScope @@ -1348,11 +1322,11 @@ class PlannerScopeResolution(NativeRecord): inspection_scope: PlannerScope def __init__( self, + *, selected_scope: PlannerScope, stale_warning_scope: PlannerScope, inspection_scope: PlannerScope, ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... class PlannerWarehouseState: """Warehouse snapshot and inspection relations gathered once per plan.""" @@ -1380,72 +1354,17 @@ class PlannerWarehouseState: migration_fingerprints: MigrationFingerprintCache = ..., ) -> None: ... -class ModelMigrationRequest(NativeRecord): - """One destination model whose data should come from an earlier relation.""" - - model: CompiledModelView - discovery: MigrationDiscovery - raw_origin: str | None - origin_location: CompiledRelationLocation | None - origin_model: str | None - force: bool - identity_only: bool - def __init__( - self, - model: CompiledModelView, - discovery: MigrationDiscovery, - raw_origin: str | None = ..., - origin_location: CompiledRelationLocation | None = ..., - origin_model: str | None = ..., - force: bool = ..., - identity_only: bool = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationRequest: ... - -class ModelMigrationDeclaration(NativeRecord): - """One project model's explicit migrate_from, resolved to a relation.""" - - model_name: str - origin_location: CompiledRelationLocation - origin_model: str | None - def __init__( - self, model_name: str, origin_location: CompiledRelationLocation, origin_model: str | None - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationDeclaration: ... - -class ModelMigrationDiscovery(NativeRecord): - """Manual and automatically discovered migration requests for one plan.""" - - requests: tuple[ModelMigrationRequest, ...] - warnings: tuple[PlanWarning, ...] - destination_fingerprints: dict[str, tuple[str, ...]] - def __init__( - self, - requests: tuple[ModelMigrationRequest, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - destination_fingerprints: dict[str, tuple[str, ...]] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationDiscovery: ... - -class MigrationCompatibilityResult(NativeRecord): - """Compatibility status plus human-readable blocking findings.""" - - status: MigrationCompatibility - findings: tuple[str, ...] - def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... - def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... - -class PlannerIdentityContext(NativeRecord): +class PlannerIdentityContext: """Expected version identities for inspection and stale-warning scopes.""" version_identities: DirectModelVersionIdentities stale_warning_identities: DirectModelVersionIdentities def __init__( self, + *, version_identities: DirectModelVersionIdentities, stale_warning_identities: DirectModelVersionIdentities, ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... class NativeModelInput: @property @@ -7027,15 +6946,6 @@ def build_migration_fingerprint( *, query_sql: str, metadata_json: str, ref_identities: Mapping[str, str], dialect: str | None ) -> str | None: ... def migration_fingerprint_ref_names(*, query_sql: str, metadata_json: str) -> frozenset[str]: ... -def check_migration_compatibility( - *, - model: CompiledModelView, - origin_relation: RelationInfo, - origin_columns: tuple[ColumnInfo, ...], - sql_analysis_enabled: bool, - column_dialect: str | None, - snapshots_config: SnapshotsConfig, -) -> MigrationCompatibilityResult: ... def migration_incremental_blocking_findings( *, model: CompiledModelView, @@ -7049,48 +6959,7 @@ def migration_snapshot_blocking_findings( dialect: str | None, snapshots_config: SnapshotsConfig, ) -> tuple[str, ...]: ... -def match_migration_fingerprints( - *, - models: tuple[CompiledModelView, ...], - new_names: frozenset[str], - seed_variants: tuple[dict[str, str], ...], - candidates: tuple[Fingerprint, ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> tuple[tuple[dict[str, str], ...], dict[str, Fingerprint], dict[str, str]]: ... -def migration_destination_fingerprints( - *, - requests: tuple[ModelMigrationRequest, ...], - variants: tuple[dict[str, str], ...], - metadata_jsons: dict[str, str], - dialect: str | None, - fingerprints: MigrationFingerprintCache, -) -> dict[str, tuple[str, ...]]: ... -def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... -def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... -def discover_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - manual_requests: tuple[ModelMigrationRequest, ...], - declarations: tuple[ModelMigrationDeclaration, ...], - state: MigrationStateInspection, - project_schemas: set[str], - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationDiscovery: ... -def decide_model_migration( - *, - runtime: PlannerRuntime, - request: ModelMigrationRequest, - origin: CompiledRelationLocation, - origin_model: str | None, - snapshot: WarehouseSnapshot, - state: MigrationStateInspection, - snapshots_config: SnapshotsConfig, -) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: ... def plan_column_migrations( *, runtime: PlannerRuntime, @@ -12300,7 +12169,7 @@ class PlannerScope(NativeRecord): sql_test_selection: SqlTestSelection = ..., ) -> None: ... -class PlannerRelationsContext(NativeRecord): +class PlannerRelationsContext: """Resolved relation and source inputs for plan entry construction.""" model_locations: dict[str, CompiledRelationLocation] @@ -12314,6 +12183,7 @@ class PlannerRelationsContext(NativeRecord): listed_source_names: frozenset[str] def __init__( self, + *, model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], function_locations: dict[str, CompiledRelationLocation], diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index 31823edc7c..3150af5573 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -4,7 +4,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( ChangeDetectionResult, ChangeKind, @@ -69,7 +68,6 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem model_version_hashes={"orders": test_case.stale_warning_hash}, ) - monkeypatch.setattr(native_module, "detect_model_changes_in_scope", lambda **_: {}) stale_scope: PlannerScope = PlannerScope( upstream_deps={}, downstream_deps={}, @@ -81,7 +79,10 @@ def test_given_execution_change_when_checking_reuse_then_requires_equivalent_sem reused: PlannerChangeResults = detect_stale_warning_changes( project=SimpleNamespace( - settings=SimpleNamespace(query_change_tracking=test_case.query_change_tracking) + settings=SimpleNamespace(query_change_tracking=test_case.query_change_tracking), + functions=(), + hook_functions=(), + sql_analysis_dialect="duckdb", ), scopes=PlannerScopeResolution( selected_scope=stale_scope, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 88f507dd64..101fbd5829 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -10,6 +10,7 @@ CompiledRelationLocation, CompileModelConfig, ModelPlanContext, + SourceEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -34,16 +35,21 @@ def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> Compile ) -def build_empty_model_plan_context() -> ModelPlanContext: - """Build an empty model-resolution context.""" +def build_empty_model_plan_context( + *, + model_locations: dict[str, CompiledRelationLocation] | None = None, + seed_locations: dict[str, CompiledRelationLocation] | None = None, + source_map: dict[str, SourceEntry] | None = None, +) -> ModelPlanContext: + """Build a model-resolution context with only the given relations.""" return ModelPlanContext( - model_locations={}, + model_locations=model_locations or {}, models_by_name={}, functions_by_name={}, - seed_locations={}, + seed_locations=seed_locations or {}, function_locations={}, - source_map={}, + source_map=source_map or {}, source_warehouse_columns={}, star_exclude_keyword="EXCLUDE", ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index c00449ebb0..461182ea37 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -2,8 +2,6 @@ from __future__ import annotations -from dataclasses import replace - import pytest from sqlbuild._native import ( @@ -25,8 +23,7 @@ ) _PYTHON_HOOK: PythonHookEntry = PythonHookEntry(name="refresh_lookup", kwargs={}) -_CONTEXT: ModelPlanContext = replace( - build_empty_model_plan_context(), +_CONTEXT: ModelPlanContext = build_empty_model_plan_context( model_locations={"orders": build_target("staging.orders", "orders")}, seed_locations={"country_codes": build_target("seeds.country_codes", "country_codes")}, source_map={"raw_orders": SourceEntry(name="raw_orders", schema="raw_prod", table="orders")},